TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { dateWords, makeSale, md, money, splitMarkdownItems, vehicleAttributes } from '../_carlib/index.js';56const BASE = 'https://www.mecum.com';7const PARSER_VERSION = '1.0.0';89export const LotSchema = z.object({ lotId: z.string(), url: z.string(), title: z.string(), lotNumber: z.string().nullable(), sold: z.boolean(), priceText: z.string().nullable(), subtitle: z.string().nullable(), image: z.string().nullable() });10export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), auctionSlug: z.string(), auctionName: z.string().nullable(), dateText: z.string().nullable(), startDate: z.string().nullable().optional(), endDate: z.string().nullable().optional(), page: z.number(), lots: z.array(LotSchema) });11export type PagePayload = z.infer<typeof PagePayloadSchema>;1213/** Completed auction slugs from the /results/ page HTML (Next.js payload, escaped). Newest first as listed. */14export function parseResultsSlugs(html: string): string[] {15 const u = html.replace(/\\"/g, '"').replace(/\\\//g, '/');16 const out: string[] = [];17 for (const m of u.matchAll(/\/auctions\/([a-z0-9-]+-(?:19|20)\d{2})\//g)) if (!out.includes(m[1]!)) out.push(m[1]!);18 return out;19}2021/** Auction page HTML → { startDate, endDate } from the embedded schema.org Event (ISO strings) */22export function parseAuctionDates(html: string): { startDate: string | null; endDate: string | null } {23 const u = html.replace(/\\"/g, '"');24 return { startDate: u.match(/"startDate":"([^"]+)"/)?.[1] ?? null, endDate: u.match(/"endDate":"([^"]+)"/)?.[1] ?? null };25}2627export function parseLotsPage(markdown: string, auctionSlug: string, page: number): PagePayload {28 const header = markdown.match(/(?:^|\n)# ([^\n]+)\n\n([^\n]+)\n/);29 const chunks = splitMarkdownItems(markdown, /^\[View /m);30 const lots: z.infer<typeof LotSchema>[] = [];31 for (const c of chunks) {32 const head = c.match(/^\[View ([^\]]+)\]\((https:\/\/www\.mecum\.com\/lots\/(\d+)\/[a-z0-9-]+\/?)[^)]*\)/);33 if (!head) continue;34 const lines = c.split('\n').map((l) => l.replace(/\\+$/, '').trim()).filter(Boolean);35 const priceText = lines.find((l) => /^\$[\d,]+$/.test(l)) ?? null;36 const sold = /!\[sold\]/i.test(c);37 const lotNumber = c.match(/\nLot ([A-Z]?\d+(?:\.\d+)?)\b/)?.[1] ?? null;38 const titleIdx = lines.findIndex((l) => l.startsWith(`[${head[1]}](`));39 const subtitle = titleIdx >= 0 ? lines.slice(titleIdx + 1).find((l) => !l.startsWith('[') && !l.startsWith('!')) ?? null : null;40 lots.push({ lotId: head[3]!, url: head[2]!, title: md.clean(head[1]!), lotNumber, sold, priceText, subtitle: subtitle ? md.clean(subtitle) : null, image: c.match(/!\[[^\]]*\]\((https:\/\/images\.mecum\.com\/[^)\s]+)\)/)?.[1] ?? null });41 }42 return { kind: 'lots_page', auctionSlug, auctionName: header?.[1]?.trim() ?? null, dateText: header?.[2]?.trim() ?? null, page, lots };43}4445const MEMORABILIA = /\b(sign|neon|gas pump|petroliana|pedal car|poster|helmet|memorabilia|collection of|display|clock|toy)\b/i;4647export class MecumConnector extends BaseConnector {48 readonly version = '1.0.0';49 readonly parserVersion = PARSER_VERSION;50 protected override minIntervalMs = 2000;5152 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {53 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 1);54 const lotPages = Number(this.meta.config.lotPagesPerAuction ?? 10);55 const progress = (ctx.options.cursor?.progress ?? {}) as Record<string, number | 'done'>;56 await this.throttle();57 const list = await ctx.fetch(`${BASE}/results/`, { engines: ['api'], responseType: 'text', expect: ['title'], parse: (r) => (r.html ? { title: parseResultsSlugs(r.html)[0] ?? null } : null) });58 if (!list.success || !list.html) {59 ctx.anomaly('page_fetch_failed', `${BASE}/results/: ${list.error ?? list.httpStatus}`);60 return;61 }62 const skipUntil = (ctx.options.cursor?.skipUntil ?? {}) as Record<string, string>;63 const nowIso = new Date().toISOString();64 const slugs = parseResultsSlugs(list.html).filter((s) => progress[s] !== 'done' && !(skipUntil[s] && skipUntil[s]! > nowIso));65 let count = 0;66 let processed = 0;67 for (const slug of slugs) {68 if (processed >= auctionsPerRun || ctx.signal?.aborted) break;69 // Auction page (plain HTTPS, free) carries schema.org Event dates: skip sales not yet held.70 await this.throttle();71 const ap = await ctx.fetch(`${BASE}/auctions/${slug}/`, { engines: ['api'], responseType: 'text', expect: ['date'], parse: (r) => (r.html ? { date: parseAuctionDates(r.html).startDate } : null), minQuality: 0 });72 const dates = ap.html ? parseAuctionDates(ap.html) : { startDate: null, endDate: null };73 if (dates.startDate && new Date(dates.startDate).getTime() > Date.now()) {74 skipUntil[slug] = dates.startDate;75 await ctx.setCursor({ progress, skipUntil, updatedAt: new Date().toISOString() });76 ctx.log.info({ slug, start: dates.startDate }, 'mecum auction not yet held; skipping');77 continue;78 }79 const startPage = typeof progress[slug] === 'number' ? (progress[slug] as number) : 1;80 let page = startPage;81 let finished = false;82 for (; page < startPage + lotPages; page++) {83 if (ctx.signal?.aborted || this.reached(ctx, count)) break;84 const url = `${BASE}/auctions/${slug}/lots/?page=${page}`;85 await this.throttle();86 const res = await ctx.fetch(url, {87 engines: ['firecrawl'],88 waitForMs: 9000,89 expect: ['title', 'price', 'status'],90 parse: (r) => {91 const p = r.markdown ? parseLotsPage(r.markdown, slug, page) : null;92 const f = p?.lots.find((l) => l.sold && l.priceText);93 return f ? { title: f.title, price: money(f.priceText, 'USD')?.amount ?? null, status: 'sold' } : p?.lots.length ? { title: p.lots[0]!.title } : null;94 },95 minQuality: 0.3,96 });97 if (!res.success || !res.markdown) {98 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);99 break;100 }101 const payload = { ...parseLotsPage(res.markdown, slug, page), startDate: dates.startDate, endDate: dates.endDate };102 if (payload.lots.length === 0) {103 finished = true;104 break;105 }106 count++;107 yield { url, externalId: `auction:${slug}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };108 if (!res.markdown.includes(`/auctions/${slug}/lots/?page=${page + 1}`)) {109 finished = true;110 page++;111 break;112 }113 }114 progress[slug] = finished ? 'done' : page;115 processed++;116 await ctx.setCursor({ progress, skipUntil, updatedAt: new Date().toISOString() });117 }118 }119120 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {121 const p = PagePayloadSchema.parse(raw.payload);122 const year = Number(p.auctionSlug.match(/(19|20)\d{2}$/)?.[0] ?? NaN);123 const saleDate = (p.startDate ? new Date(p.startDate) : null) ?? dateWords(p.dateText ?? '', Number.isFinite(year) ? year : null);124 if (!saleDate) return [];125 const out: NormalizedSale[] = [];126 for (const lot of p.lots) {127 if (!lot.sold || !lot.priceText) continue;128 const m = money(lot.priceText, 'USD');129 if (!m) continue;130 const memorabilia = MEMORABILIA.test(lot.title) && !/^\d{4}\s/.test(lot.title);131 const attributes = vehicleAttributes(lot.title, { country: 'US', identifiers: { mecum_lot_id: lot.lotId }, metadata: { auction: p.auctionName ?? p.auctionSlug, auction_dates: p.dateText, auction_start: p.startDate ?? null, auction_end: p.endDate ?? null, highlights: lot.subtitle }, ...(memorabilia ? { categorySlug: 'automotive_memorabilia' as const } : {}), moto: /motorcycle/i.test(p.auctionSlug) });132 out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: lot.lotId, rawTitle: lot.title, attributes, price: m.amount, currency: 'USD', saleDate, buyerPremiumIncluded: null, auctionHouse: 'Mecum Auctions', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], description: lot.subtitle, location: 'US', observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, confidence: 0.85 }));133 }134 return out;135 }136}137138export default (meta: ConnectorMeta) => new MecumConnector(meta);139