import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared'; import { whiskyFacts } from '../scotch-whisky-auctions/index.js'; import { dateDMonY, isBundleTitle, moneyNum } from '../_g10-lib/index.js'; /** * Whisky.Auction (London) — past auction lot grids. Plain server-rendered HTML (no anti-bot for the * honest bot UA): /auctions lists every past auction; /auctions/auction/?category=…&pageIndex=N&pageSize=90 * &sort=Price&dir=desc lists the lots with the winning bid (GBP hammer, buyer commission separate) and the * lot end date in data-attributes. Sold vs unsold comes from the lot CSS state (Met / NA / Aftersale vs * NotMet, winning bid -1). One raw record per grid page; one sale per sold lot. */ const BASE = 'https://whisky.auction'; const PARSER_VERSION = '1.0.0'; export const SeedSchema = z.object({ category: z.string(), slug: z.enum(['whisky', 'rum', 'cognac', 'wine']) }); export type Seed = z.infer; export const AuctionEntrySchema = z.object({ id: z.string(), title: z.string(), startedOn: z.string().nullable(), endedOn: z.string().nullable(), highestBid: z.number().nullable(), image: z.string().nullable() }); export type AuctionEntry = z.infer; export const LotSchema = z.object({ lotId: z.string(), productId: z.string().nullable(), url: z.string(), title: z.string(), sub1: z.string().nullable(), sub2: z.string().nullable(), image: z.string().nullable(), winningBid: z.number().nullable(), status: z.enum(['met', 'na', 'aftersale', 'notmet', 'unknown']), endDate: z.string().nullable(), endedOn: z.string().nullable(), vatScheme: z.string().nullable(), }); export type Lot = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('lot_page'), url: z.string(), auctionId: z.string(), auctionTitle: z.string().nullable(), seed: SeedSchema, pageIndex: z.number().int(), pageSize: z.number().int(), total: z.number().int().nullable(), lots: z.array(LotSchema), /** trimmed raw HTML kept for parser tests (optional) */ snapshot: z.string().optional(), }); export type PagePayload = z.infer; const ConfigSchema = z.object({ seeds: z.array(SeedSchema).min(1), pageSize: z.number().int().min(30).max(90).default(90), auctionsPerRun: z.number().int().min(1).default(1), pagesPerRun: z.number().int().min(1).default(12), }); /** /auctions → past auction entries (newest first). */ export function parseAuctionList(htmlText: string): AuctionEntry[] { const $ = H.load(htmlText); const out: AuctionEntry[] = []; $('.auctionentry').each((_, el) => { const a = $(el).find('a[href*="/auctions/auction/"]').first(); const id = (a.attr('href') ?? '').match(/\/auctions\/auction\/(\d+)/)?.[1]; if (!id) return; const title = H.text($(el).find('.auctionentry-title-main').first()) ?? `Auction ${id}`; out.push({ id, title, startedOn: H.text($(el).find('.auction-date-start .value').first()), endedOn: H.text($(el).find('.auction-date-end .value').first()), highestBid: moneyNum(H.text($(el).find('.bid-highest .value').first())), image: $(el).find('img.auctionentry-image').attr('src') ?? null, }); }); return out.sort((a, b) => Number(b.id) - Number(a.id)); } function statusFromClass(cls: string): Lot['status'] { if (/\bAftersale\b/.test(cls)) return 'aftersale'; if (/\bNotMet\b/.test(cls)) return 'notmet'; if (/\bMet\b/.test(cls)) return 'met'; if (/\bNA\b/.test(cls)) return 'na'; return 'unknown'; } /** One lot grid page → total count + lots. */ export function parseLotPage(htmlText: string): { total: number | null; lots: Lot[]; auctionTitle: string | null } { const $ = H.load(htmlText); const showing = $('.detail-showing span').map((_, s) => $(s).text().replace(/[^0-9]/g, '')).get().filter(Boolean); const total = showing.length ? Number(showing[showing.length - 1]) : null; const auctionTitle = H.text($('h1').first())?.replace(/\s+/g, ' ') ?? H.text($('title').first())?.split(/\s+-\s+Past Auctions|\s*\|/)[0]?.trim() ?? null; const lots: Lot[] = []; $('div.lot.lotItem').each((_, el) => { const e = $(el); const lotId = (e.attr('id') ?? '').replace(/^lot_/, ''); if (!/^\d+$/.test(lotId)) return; const href = e.find('a[href^="/auctions/lot/"]').first().attr('href'); const title = H.text(e.find('.lotName1').first()); if (!href || !title) return; const img = e.find('img.lot-image').first(); const srcset = img.attr('srcset') ?? ''; const big = srcset.match(/(https:\/\/media\.whisky\.auction\/360\/[^\s,]+)/)?.[1] ?? img.attr('src') ?? null; const bidRaw = e.attr('data-winningbid'); const bid = bidRaw !== undefined ? Number.parseFloat(bidRaw) : NaN; lots.push({ lotId, productId: e.attr('data-product-id') ?? null, url: `${BASE}${href}`, title, sub1: H.text(e.find('.lotName2').first()), sub2: H.text(e.find('.lotName3').first()), image: big, winningBid: Number.isFinite(bid) ? bid : null, status: statusFromClass(e.attr('class') ?? ''), endDate: e.attr('data-enddate') ?? null, endedOn: H.text(e.find('.ended-date').first()), vatScheme: e.attr('data-vat-scheme') ?? null, }); }); return { total, lots, auctionTitle }; } export function pageUrl(auctionId: string, seed: Seed, pageIndex: number, pageSize: number): string { const q = new URLSearchParams({ category: seed.category, pageIndex: String(pageIndex), pageSize: String(pageSize), sort: 'Price', dir: 'desc' }); return `${BASE}/auctions/auction/${auctionId}?${q.toString()}`; } /** "2026-05-19 21:20:00" (site clock, UK) → Date (UTC of the same wall-clock instant; the day is what matters). */ export function parseEndDate(s: string | null | undefined): Date | null { const m = s?.match(/^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2})(?::(\d{2}))?)?/); if (!m) return null; const d = new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4] ?? 0), Number(m[5] ?? 0), Number(m[6] ?? 0))); return Number.isNaN(d.getTime()) ? null : d; } export function isSold(lot: Lot): boolean { return lot.status !== 'notmet' && lot.winningBid !== null && lot.winningBid > 0; } /** Trim a grid page's HTML to the first `n` lot cards (fixture snapshots). */ export function trimLotHtml(htmlText: string, n = 3): string { const $ = H.load(htmlText); const lots = $('div.lot.lotItem').toArray().slice(0, n).map((el) => $.html(el)); const showing = $('.detail-showing').first(); const h1 = $('h1').first(); return `${$('title').text()}

${h1.html() ?? ''}

${lots.join('\n')}
`; } interface Cursor { progress?: Record; complete?: string[]; } export class WhiskyAuctionConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2500; private readonly cfg: z.infer; constructor(meta: ConnectorMeta) { super(meta); this.cfg = ConfigSchema.parse(meta.config); } async *crawl(ctx: CrawlContext): AsyncIterable { const cursor = (ctx.options.cursor ?? {}) as Cursor; const progress: Record = { ...(cursor.progress ?? {}) }; const complete = new Set(cursor.complete ?? []); const backfill = ctx.options.mode === 'backfill'; const perRun = backfill ? Math.max(this.cfg.auctionsPerRun, 3) : this.cfg.auctionsPerRun; const pageBudget = backfill ? Math.min(this.policy.backfillMaxPages, this.cfg.pagesPerRun * 4) : this.cfg.pagesPerRun; await this.throttle(); const list = await ctx.fetch(`${BASE}/auctions`, { engines: ['api'], responseType: 'text', minQuality: 0 }); const entries = list.success && list.html ? parseAuctionList(list.html) : []; if (!entries.length) { ctx.anomaly('page_fetch_failed', `auction list: ${list.error ?? list.httpStatus}`); return; } // Only auctions whose end date is in the past are settled (the list is "Past Auctions", but guard anyway). const ended = entries.filter((e) => { const d = dateDMonY(e.endedOn); return !d || d.getTime() < Date.now() - 12 * 3600_000; }); const candidates = ended.filter((e) => !complete.has(e.id)).slice(0, perRun); let pages = 0; let count = 0; let items = 0; for (const auction of candidates) { const state = progress[auction.id] ?? { seedIndex: 0, pageIndex: 0 }; let seedIndex = state.seedIndex; let pageIndex = state.pageIndex; let auctionDone = false; while (seedIndex < this.cfg.seeds.length && pages < pageBudget && !ctx.signal?.aborted && !this.reached(ctx, count)) { const seed = this.cfg.seeds[seedIndex]!; const url = pageUrl(auction.id, seed, pageIndex, this.cfg.pageSize); await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'currency', 'date'], parse: (r) => { const p = r.html ? parseLotPage(r.html) : null; if (!p) return null; const sold = p.lots.find(isSold); return { title: p.lots[0]?.title ?? (p.total === 0 ? 'empty' : null), price: sold?.winningBid ?? null, currency: sold ? 'GBP' : null, date: sold?.endDate ?? null }; }, minQuality: 0.2, }); pages++; const parsed = res.success && res.html ? parseLotPage(res.html) : null; if (!parsed) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const allUnsold = parsed.lots.length > 0 && parsed.lots.every((l) => !isSold(l)); if (parsed.lots.length) { count++; items += parsed.lots.length; const payload: PagePayload = { kind: 'lot_page', url, auctionId: auction.id, auctionTitle: parsed.auctionTitle ?? auction.title, seed, pageIndex, pageSize: this.cfg.pageSize, total: parsed.total, lots: parsed.lots }; yield { url, externalId: `auction:${auction.id}:${seed.slug}:p${pageIndex}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } // Sorted by price desc: once a whole page is unsold (or the grid is exhausted) the seed is finished. const exhausted = parsed.lots.length < this.cfg.pageSize || (parsed.total !== null && (pageIndex + 1) * this.cfg.pageSize >= parsed.total) || allUnsold || parsed.lots.length === 0; if (exhausted) { seedIndex++; pageIndex = 0; } else { pageIndex++; } progress[auction.id] = { seedIndex, pageIndex }; await ctx.setCursor({ progress, complete: [...complete] }); await ctx.progress({ page: pages, itemsProcessed: items }); } if (seedIndex >= this.cfg.seeds.length) { auctionDone = true; complete.add(auction.id); delete progress[auction.id]; } await ctx.setCursor({ progress, complete: [...complete] }); if (!auctionDone) break; // page budget exhausted; resume this auction next run } if (backfill && ended.every((e) => complete.has(e.id))) await ctx.setCursor({ progress, complete: [...complete], done: true }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const lot of p.lots) { if (!isSold(lot) || !lot.winningBid) continue; const saleDate = parseEndDate(lot.endDate) ?? dateDMonY(lot.endedOn); if (!saleDate) continue; const f = whiskyFacts(lot.title); const categorySlug = p.seed.slug; const bottled = lot.sub1?.match(/\b(19|20)\d{2}\b/)?.[0]; const abv = lot.sub2?.match(/(\d{1,2}(?:\.\d)?)\s*%/)?.[1]; const sizeFromSub = lot.sub2?.match(/(\d+(?:\.\d+)?)\s*(cl|ml|l|litre|liter)\b/i); const size = sizeFromSub ? `${sizeFromSub[1]}${sizeFromSub[2]!.toLowerCase().startsWith('l') ? 'L' : sizeFromSub[2]!.toLowerCase()}` : f.size; const attributes = AssetAttributesSchema.parse({ categorySlug, brand: f.brand, name: lot.title, year: f.vintage, size, identifiers: { whisky_auction_lot: lot.lotId, ...(lot.productId ? { whisky_auction_product: lot.productId } : {}) }, metadata: { age_statement: f.age, bottled_year: bottled ? Number(bottled) : null, abv: abv ? Number(abv) : null, subtitle: [lot.sub1, lot.sub2].filter(Boolean).join(' · ') || null, auction_id: p.auctionId, auction_title: p.auctionTitle, lot_status: lot.status, sold_in_aftersale: lot.status === 'aftersale', vat_scheme: lot.vatScheme, source_category_filter: p.seed.category, end_time_local: lot.endDate, }, }); out.push( NormalizedSaleSchema.parse({ kind: 'sale', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: lot.url, externalId: lot.lotId, rawTitle: [lot.title, lot.sub1, lot.sub2].filter(Boolean).join(' '), imageUrls: lot.image ? [lot.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: lot.status === 'aftersale' ? 0.8 : 0.88, parserVersion: PARSER_VERSION, saleType: 'auction', saleDate, price: lot.winningBid, currency: 'GBP', buyerPremiumIncluded: false, quantity: 1, isBundle: isBundleTitle(lot.title) || /\bx\s?\d|\(\d+\s?x\)|\bset of\b|\blot of\b|\b\d+\s*bottles\b/i.test(lot.title), location: 'London, United Kingdom', auctionHouse: 'Whisky.Auction', lotNumber: lot.lotId, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): WhiskyAuctionConnector { return new WhiskyAuctionConnector(meta); }