TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';4import { whiskyFacts } from '../scotch-whisky-auctions/index.js';5import { dateDMonY, isBundleTitle, moneyNum } from '../_g10-lib/index.js';67/**8 * Whisky.Auction (London) — past auction lot grids. Plain server-rendered HTML (no anti-bot for the9 * honest bot UA): /auctions lists every past auction; /auctions/auction/<id>?category=…&pageIndex=N&pageSize=9010 * &sort=Price&dir=desc lists the lots with the winning bid (GBP hammer, buyer commission separate) and the11 * lot end date in data-attributes. Sold vs unsold comes from the lot CSS state (Met / NA / Aftersale vs12 * NotMet, winning bid -1). One raw record per grid page; one sale per sold lot.13 */14const BASE = 'https://whisky.auction';15const PARSER_VERSION = '1.0.0';1617export const SeedSchema = z.object({ category: z.string(), slug: z.enum(['whisky', 'rum', 'cognac', 'wine']) });18export type Seed = z.infer<typeof SeedSchema>;1920export const AuctionEntrySchema = z.object({ id: z.string(), title: z.string(), startedOn: z.string().nullable(), endedOn: z.string().nullable(), highestBid: z.number().nullable(), image: z.string().nullable() });21export type AuctionEntry = z.infer<typeof AuctionEntrySchema>;2223export const LotSchema = z.object({24 lotId: z.string(),25 productId: z.string().nullable(),26 url: z.string(),27 title: z.string(),28 sub1: z.string().nullable(),29 sub2: z.string().nullable(),30 image: z.string().nullable(),31 winningBid: z.number().nullable(),32 status: z.enum(['met', 'na', 'aftersale', 'notmet', 'unknown']),33 endDate: z.string().nullable(),34 endedOn: z.string().nullable(),35 vatScheme: z.string().nullable(),36});37export type Lot = z.infer<typeof LotSchema>;3839export const PagePayloadSchema = z.object({40 kind: z.literal('lot_page'),41 url: z.string(),42 auctionId: z.string(),43 auctionTitle: z.string().nullable(),44 seed: SeedSchema,45 pageIndex: z.number().int(),46 pageSize: z.number().int(),47 total: z.number().int().nullable(),48 lots: z.array(LotSchema),49 /** trimmed raw HTML kept for parser tests (optional) */50 snapshot: z.string().optional(),51});52export type PagePayload = z.infer<typeof PagePayloadSchema>;5354const ConfigSchema = z.object({55 seeds: z.array(SeedSchema).min(1),56 pageSize: z.number().int().min(30).max(90).default(90),57 auctionsPerRun: z.number().int().min(1).default(1),58 pagesPerRun: z.number().int().min(1).default(12),59});6061/** /auctions → past auction entries (newest first). */62export function parseAuctionList(htmlText: string): AuctionEntry[] {63 const $ = H.load(htmlText);64 const out: AuctionEntry[] = [];65 $('.auctionentry').each((_, el) => {66 const a = $(el).find('a[href*="/auctions/auction/"]').first();67 const id = (a.attr('href') ?? '').match(/\/auctions\/auction\/(\d+)/)?.[1];68 if (!id) return;69 const title = H.text($(el).find('.auctionentry-title-main').first()) ?? `Auction ${id}`;70 out.push({71 id,72 title,73 startedOn: H.text($(el).find('.auction-date-start .value').first()),74 endedOn: H.text($(el).find('.auction-date-end .value').first()),75 highestBid: moneyNum(H.text($(el).find('.bid-highest .value').first())),76 image: $(el).find('img.auctionentry-image').attr('src') ?? null,77 });78 });79 return out.sort((a, b) => Number(b.id) - Number(a.id));80}8182function statusFromClass(cls: string): Lot['status'] {83 if (/\bAftersale\b/.test(cls)) return 'aftersale';84 if (/\bNotMet\b/.test(cls)) return 'notmet';85 if (/\bMet\b/.test(cls)) return 'met';86 if (/\bNA\b/.test(cls)) return 'na';87 return 'unknown';88}8990/** One lot grid page → total count + lots. */91export function parseLotPage(htmlText: string): { total: number | null; lots: Lot[]; auctionTitle: string | null } {92 const $ = H.load(htmlText);93 const showing = $('.detail-showing span').map((_, s) => $(s).text().replace(/[^0-9]/g, '')).get().filter(Boolean);94 const total = showing.length ? Number(showing[showing.length - 1]) : null;95 const auctionTitle = H.text($('h1').first())?.replace(/\s+/g, ' ') ?? H.text($('title').first())?.split(/\s+-\s+Past Auctions|\s*\|/)[0]?.trim() ?? null;96 const lots: Lot[] = [];97 $('div.lot.lotItem').each((_, el) => {98 const e = $(el);99 const lotId = (e.attr('id') ?? '').replace(/^lot_/, '');100 if (!/^\d+$/.test(lotId)) return;101 const href = e.find('a[href^="/auctions/lot/"]').first().attr('href');102 const title = H.text(e.find('.lotName1').first());103 if (!href || !title) return;104 const img = e.find('img.lot-image').first();105 const srcset = img.attr('srcset') ?? '';106 const big = srcset.match(/(https:\/\/media\.whisky\.auction\/360\/[^\s,]+)/)?.[1] ?? img.attr('src') ?? null;107 const bidRaw = e.attr('data-winningbid');108 const bid = bidRaw !== undefined ? Number.parseFloat(bidRaw) : NaN;109 lots.push({110 lotId,111 productId: e.attr('data-product-id') ?? null,112 url: `${BASE}${href}`,113 title,114 sub1: H.text(e.find('.lotName2').first()),115 sub2: H.text(e.find('.lotName3').first()),116 image: big,117 winningBid: Number.isFinite(bid) ? bid : null,118 status: statusFromClass(e.attr('class') ?? ''),119 endDate: e.attr('data-enddate') ?? null,120 endedOn: H.text(e.find('.ended-date').first()),121 vatScheme: e.attr('data-vat-scheme') ?? null,122 });123 });124 return { total, lots, auctionTitle };125}126127export function pageUrl(auctionId: string, seed: Seed, pageIndex: number, pageSize: number): string {128 const q = new URLSearchParams({ category: seed.category, pageIndex: String(pageIndex), pageSize: String(pageSize), sort: 'Price', dir: 'desc' });129 return `${BASE}/auctions/auction/${auctionId}?${q.toString()}`;130}131132/** "2026-05-19 21:20:00" (site clock, UK) → Date (UTC of the same wall-clock instant; the day is what matters). */133export function parseEndDate(s: string | null | undefined): Date | null {134 const m = s?.match(/^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2})(?::(\d{2}))?)?/);135 if (!m) return null;136 const d = new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4] ?? 0), Number(m[5] ?? 0), Number(m[6] ?? 0)));137 return Number.isNaN(d.getTime()) ? null : d;138}139140export function isSold(lot: Lot): boolean {141 return lot.status !== 'notmet' && lot.winningBid !== null && lot.winningBid > 0;142}143144/** Trim a grid page's HTML to the first `n` lot cards (fixture snapshots). */145export function trimLotHtml(htmlText: string, n = 3): string {146 const $ = H.load(htmlText);147 const lots = $('div.lot.lotItem').toArray().slice(0, n).map((el) => $.html(el));148 const showing = $('.detail-showing').first();149 const h1 = $('h1').first();150 return `<!doctype html><html><head><title>${$('title').text()}</title></head><body><h1 class="${h1.attr('class') ?? 'title-main'}">${h1.html() ?? ''}</h1><div class="pagination pagination-top">${showing.length ? $.html(showing) : ''}</div><div id="auctionlive" class="auction-items-grid">${lots.join('\n')}</div></body></html>`;151}152153interface Cursor {154 progress?: Record<string, { seedIndex: number; pageIndex: number }>;155 complete?: string[];156}157158export class WhiskyAuctionConnector extends BaseConnector {159 readonly version = '1.0.0';160 readonly parserVersion = PARSER_VERSION;161 protected override minIntervalMs = 2500;162 private readonly cfg: z.infer<typeof ConfigSchema>;163164 constructor(meta: ConnectorMeta) {165 super(meta);166 this.cfg = ConfigSchema.parse(meta.config);167 }168169 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {170 const cursor = (ctx.options.cursor ?? {}) as Cursor;171 const progress: Record<string, { seedIndex: number; pageIndex: number }> = { ...(cursor.progress ?? {}) };172 const complete = new Set<string>(cursor.complete ?? []);173 const backfill = ctx.options.mode === 'backfill';174 const perRun = backfill ? Math.max(this.cfg.auctionsPerRun, 3) : this.cfg.auctionsPerRun;175 const pageBudget = backfill ? Math.min(this.policy.backfillMaxPages, this.cfg.pagesPerRun * 4) : this.cfg.pagesPerRun;176177 await this.throttle();178 const list = await ctx.fetch(`${BASE}/auctions`, { engines: ['api'], responseType: 'text', minQuality: 0 });179 const entries = list.success && list.html ? parseAuctionList(list.html) : [];180 if (!entries.length) {181 ctx.anomaly('page_fetch_failed', `auction list: ${list.error ?? list.httpStatus}`);182 return;183 }184 // Only auctions whose end date is in the past are settled (the list is "Past Auctions", but guard anyway).185 const ended = entries.filter((e) => {186 const d = dateDMonY(e.endedOn);187 return !d || d.getTime() < Date.now() - 12 * 3600_000;188 });189 const candidates = ended.filter((e) => !complete.has(e.id)).slice(0, perRun);190 let pages = 0;191 let count = 0;192 let items = 0;193 for (const auction of candidates) {194 const state = progress[auction.id] ?? { seedIndex: 0, pageIndex: 0 };195 let seedIndex = state.seedIndex;196 let pageIndex = state.pageIndex;197 let auctionDone = false;198 while (seedIndex < this.cfg.seeds.length && pages < pageBudget && !ctx.signal?.aborted && !this.reached(ctx, count)) {199 const seed = this.cfg.seeds[seedIndex]!;200 const url = pageUrl(auction.id, seed, pageIndex, this.cfg.pageSize);201 await this.throttle();202 const res = await ctx.fetch(url, {203 engines: ['api'],204 responseType: 'text',205 expect: ['title', 'price', 'currency', 'date'],206 parse: (r) => {207 const p = r.html ? parseLotPage(r.html) : null;208 if (!p) return null;209 const sold = p.lots.find(isSold);210 return { title: p.lots[0]?.title ?? (p.total === 0 ? 'empty' : null), price: sold?.winningBid ?? null, currency: sold ? 'GBP' : null, date: sold?.endDate ?? null };211 },212 minQuality: 0.2,213 });214 pages++;215 const parsed = res.success && res.html ? parseLotPage(res.html) : null;216 if (!parsed) {217 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);218 break;219 }220 const allUnsold = parsed.lots.length > 0 && parsed.lots.every((l) => !isSold(l));221 if (parsed.lots.length) {222 count++;223 items += parsed.lots.length;224 const payload: PagePayload = { kind: 'lot_page', url, auctionId: auction.id, auctionTitle: parsed.auctionTitle ?? auction.title, seed, pageIndex, pageSize: this.cfg.pageSize, total: parsed.total, lots: parsed.lots };225 yield { url, externalId: `auction:${auction.id}:${seed.slug}:p${pageIndex}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };226 }227 // Sorted by price desc: once a whole page is unsold (or the grid is exhausted) the seed is finished.228 const exhausted = parsed.lots.length < this.cfg.pageSize || (parsed.total !== null && (pageIndex + 1) * this.cfg.pageSize >= parsed.total) || allUnsold || parsed.lots.length === 0;229 if (exhausted) {230 seedIndex++;231 pageIndex = 0;232 } else {233 pageIndex++;234 }235 progress[auction.id] = { seedIndex, pageIndex };236 await ctx.setCursor({ progress, complete: [...complete] });237 await ctx.progress({ page: pages, itemsProcessed: items });238 }239 if (seedIndex >= this.cfg.seeds.length) {240 auctionDone = true;241 complete.add(auction.id);242 delete progress[auction.id];243 }244 await ctx.setCursor({ progress, complete: [...complete] });245 if (!auctionDone) break; // page budget exhausted; resume this auction next run246 }247 if (backfill && ended.every((e) => complete.has(e.id))) await ctx.setCursor({ progress, complete: [...complete], done: true });248 }249250 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {251 const p = PagePayloadSchema.parse(raw.payload);252 const out: NormalizedRecord[] = [];253 for (const lot of p.lots) {254 if (!isSold(lot) || !lot.winningBid) continue;255 const saleDate = parseEndDate(lot.endDate) ?? dateDMonY(lot.endedOn);256 if (!saleDate) continue;257 const f = whiskyFacts(lot.title);258 const categorySlug = p.seed.slug;259 const bottled = lot.sub1?.match(/\b(19|20)\d{2}\b/)?.[0];260 const abv = lot.sub2?.match(/(\d{1,2}(?:\.\d)?)\s*%/)?.[1];261 const sizeFromSub = lot.sub2?.match(/(\d+(?:\.\d+)?)\s*(cl|ml|l|litre|liter)\b/i);262 const size = sizeFromSub ? `${sizeFromSub[1]}${sizeFromSub[2]!.toLowerCase().startsWith('l') ? 'L' : sizeFromSub[2]!.toLowerCase()}` : f.size;263 const attributes = AssetAttributesSchema.parse({264 categorySlug,265 brand: f.brand,266 name: lot.title,267 year: f.vintage,268 size,269 identifiers: { whisky_auction_lot: lot.lotId, ...(lot.productId ? { whisky_auction_product: lot.productId } : {}) },270 metadata: {271 age_statement: f.age,272 bottled_year: bottled ? Number(bottled) : null,273 abv: abv ? Number(abv) : null,274 subtitle: [lot.sub1, lot.sub2].filter(Boolean).join(' · ') || null,275 auction_id: p.auctionId,276 auction_title: p.auctionTitle,277 lot_status: lot.status,278 sold_in_aftersale: lot.status === 'aftersale',279 vat_scheme: lot.vatScheme,280 source_category_filter: p.seed.category,281 end_time_local: lot.endDate,282 },283 });284 out.push(285 NormalizedSaleSchema.parse({286 kind: 'sale',287 connectorId: this.meta.id,288 sourceId: this.meta.sourceId,289 sourceUrl: lot.url,290 externalId: lot.lotId,291 rawTitle: [lot.title, lot.sub1, lot.sub2].filter(Boolean).join(' '),292 imageUrls: lot.image ? [lot.image] : [],293 attributes,294 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },295 condition: { condition: null, conditionRaw: null, completeness: null },296 observedAt: raw.fetchedAt,297 confidence: lot.status === 'aftersale' ? 0.8 : 0.88,298 parserVersion: PARSER_VERSION,299 saleType: 'auction',300 saleDate,301 price: lot.winningBid,302 currency: 'GBP',303 buyerPremiumIncluded: false,304 quantity: 1,305 isBundle: isBundleTitle(lot.title) || /\bx\s?\d|\(\d+\s?x\)|\bset of\b|\blot of\b|\b\d+\s*bottles\b/i.test(lot.title),306 location: 'London, United Kingdom',307 auctionHouse: 'Whisky.Auction',308 lotNumber: lot.lotId,309 }),310 );311 }312 return out;313 }314}315316export default function createConnector(meta: ConnectorMeta): WhiskyAuctionConnector {317 return new WhiskyAuctionConnector(meta);318}319