import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { parseSourceDate, AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared'; /** * Scotch Whisky Auctions — monthly online whisky auctions (Glasgow) with a public archive of * results. One raw record per lot-list page; normalise → one sale per lot with a "Sold for" price. */ const BASE = 'https://www.scotchwhiskyauctions.com'; const PARSER_VERSION = '1.0.0'; export const AuctionSchema = z.object({ id: z.string(), slug: z.string(), title: z.string(), endedOn: z.string().nullable(), lotCount: z.number().nullable(), ended: z.boolean() }); export type Auction = z.infer; export const LotSchema = z.object({ itemId: z.string(), url: z.string(), title: z.string(), lotNo: z.string().nullable(), soldText: z.string().nullable(), priceGbp: z.number().nullable(), image: z.string().nullable() }); export const PagePayloadSchema = z.object({ kind: z.literal('lot_page'), url: z.string(), auction: AuctionSchema, page: z.number(), totalPages: z.number().nullable(), lots: z.array(LotSchema) }); export type PagePayload = z.infer; /** Parse /auctions/ : cards "The 181st Auction · Ended July 12, 2026 · There are 4926 lots in this auction". */ export function parseAuctionList(htmlText: string): Auction[] { const $ = H.load(htmlText); const out: Auction[] = []; $('a.auction').each((_, a) => { const href = $(a).attr('href') ?? ''; const m = href.match(/^\/auctions\/(\d+)-([^/]+)\/?$/); if (!m) return; const title = H.text($(a).find('h4')) ?? ''; const status = H.text($(a).find('h5')) ?? ''; const lots = H.text($(a).find('h6'))?.match(/(\d[\d,]*)\s+lots/i)?.[1] ?? null; const ended = /^Ended\b/i.test(status); const dateTxt = status.replace(/^(Ended|Ends)\s*/i, ''); const d = parseSourceDate(dateTxt); out.push({ id: m[1]!, slug: m[2]!, title, endedOn: d ? d.toISOString() : null, lotCount: lots ? Number(lots.replace(/,/g, '')) : null, ended }); }); return out; } /** Parse an auction lot-list page (20 lots): title, lot number, "Sold for £100 in July 2026", image. */ export function parseLotPage(htmlText: string, url: string, auction: Auction, page: number): PagePayload { const $ = H.load(htmlText); const totalPages = Number(htmlText.match(/Page \d+ of (\d+)/)?.[1]) || null; const lots: z.infer[] = []; $('a.lot').each((_, a) => { const href = $(a).attr('href') ?? ''; const m = href.match(/\/auctions\/\d+-[^/]+\/(\d+)-[^/]*\/?$/); if (!m) return; const title = H.text($(a).find('h4')); if (!title) return; const lotNo = H.text($(a).find('h6'))?.replace(/^Lot no\s*/i, '') ?? null; const soldText = H.text($(a).find('p.sold')) ?? null; const price = soldText?.match(/Sold for £([\d,]+(?:\.\d+)?)/i)?.[1] ?? null; const bg = $(a).find('.aucimg').attr('style') ?? ''; const image = bg.match(/url\('([^']+)'\)/)?.[1] ?? null; lots.push({ itemId: m[1]!, url: BASE + href.replace(/\/?$/, '/'), title, lotNo, soldText, priceGbp: price ? Number(price.replace(/,/g, '')) : null, image }); }); return { kind: 'lot_page', url, auction, page, totalPages, lots }; } const SIZE_RE = /(\d+(?:\.\d+)?)\s?(cl|ml|l|litre|liter)s?\b/i; const AGE_RE = /(\d{1,2})\s*[- ]?\s*(?:year|yo\b|y\.?o\.?)/i; const VINTAGE_RE = /\b(19[2-9]\d|20[0-2]\d)\b(?!\s*(?:year|yo))/i; export interface WhiskyFacts { brand: string | null; age: number | null; vintage: number | null; size: string | null; categorySlug: 'whisky' | 'rum' | 'cognac'; } /** Heuristic facts from a lot title ("Macallan 1989 18 Year Old Gran Reserva 70cl"). Unknown → null, never guessed. */ export function whiskyFacts(title: string): WhiskyFacts { const t = title.replace(/[‘’]/g, "'"); const categorySlug: WhiskyFacts['categorySlug'] = /\brum\b|\brhum\b/i.test(t) ? 'rum' : /\bcognac\b|\barmagnac\b/i.test(t) ? 'cognac' : 'whisky'; const age = t.match(AGE_RE)?.[1] ? Number(t.match(AGE_RE)![1]) : null; const vintage = t.match(VINTAGE_RE)?.[1] ? Number(t.match(VINTAGE_RE)![1]) : null; const sizeM = t.match(SIZE_RE); const size = sizeM ? `${sizeM[1]}${sizeM[2]!.toLowerCase().startsWith('l') ? 'L' : sizeM[2]!.toLowerCase()}` : null; // Brand = leading words before the first digit / age / "Year" / "-" ; keep 1–4 words const lead = t.split(/\s+(?=\d)|\s+-\s+|\s+\(/)[0] ?? t; const words = lead.replace(/^['"]|['"]$/g, '').trim().split(/\s+/).filter(Boolean); const brand = words.length ? words.slice(0, Math.min(words.length, 4)).join(' ') : null; return { brand: brand && /[a-z]/i.test(brand) ? brand : null, age, vintage, size, categorySlug }; } export class ScotchWhiskyAuctionsConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; override readonly urlPatterns = [/scotchwhiskyauctions\.com\/auctions\/\d+-[^/]+\/\d+-/]; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const listUrl = String(this.meta.config.auctionsUrl ?? `${BASE}/auctions/`); const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2); const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 12); const cursor = (ctx.options.cursor ?? {}) as { progress?: Record; complete?: string[] }; const progress: Record = { ...(cursor.progress ?? {}) }; const complete = new Set(cursor.complete ?? []); const list = await ctx.fetch(listUrl, { engines: ['api'], responseType: 'text', minQuality: 0 }); if (!list.success || !list.html) { ctx.anomaly('auction_list_failed', list.error ?? String(list.httpStatus)); return; } let auctions = parseAuctionList(list.html).filter((a) => a.ended); // newest first; incremental = newest auctions not yet complete, backfill = continue with older ones auctions.sort((a, b) => Number(b.id) - Number(a.id)); if (ctx.options.mode !== 'backfill') auctions = auctions.slice(0, 6); let count = 0; let pagesFetched = 0; let auctionsTouched = 0; for (const auction of auctions) { if (complete.has(auction.id)) continue; if (auctionsTouched >= auctionsPerRun) break; auctionsTouched++; let page = (progress[auction.id] ?? 0) + 1; for (;;) { if (ctx.signal?.aborted || this.reached(ctx, count) || pagesFetched >= pagesPerRun) return void (await ctx.setCursor({ progress, complete: [...complete] })); const url = `${BASE}/auctions/${auction.id}-${auction.slug}/${page > 1 ? `?page=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'currency', 'status'], parse: (r) => { const p = r.html ? parseLotPage(r.html, url, auction, page) : null; const sold = p?.lots.find((l) => l.priceGbp); return p && p.lots.length ? { title: p.lots[0]!.title, price: sold?.priceGbp ?? null, currency: sold ? 'GBP' : null, status: sold ? 'sold' : null } : null; }, }); pagesFetched++; const payload = res.success && res.html ? parseLotPage(res.html, url, auction, page) : null; if (!payload || payload.lots.length === 0) { ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); complete.add(auction.id); break; } count++; yield { url, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; progress[auction.id] = page; if (payload.totalPages && page >= payload.totalPages) { complete.add(auction.id); break; } page++; } await ctx.setCursor({ progress, complete: [...complete] }); } } async lookup(url: string, ctx: CrawlContext): Promise { const m = url.match(/\/auctions\/(\d+)-([^/]+)\/(\d+)-/); if (!m) return []; const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 }); if (!res.success || !res.html) return []; const $ = H.load(res.html); const title = H.text($('h1').first()) ?? H.text($('h4').first()); const soldText = res.html.match(/Sold for £[\d,]+(?:\.\d+)?[^<]*/)?.[0] ?? null; const lotNo = res.html.match(/Lot no\s*([\d-]+)/i)?.[1] ?? null; const ended = res.html.match(/Ended\s+([A-Z][a-z]+ \d{1,2}, \d{4})/)?.[1] ?? null; const endedOn = ended ? parseSourceDate(ended)?.toISOString() ?? null : null; if (!title) return []; const price = soldText?.match(/£([\d,]+(?:\.\d+)?)/)?.[1] ?? null; const image = $('meta[property="og:image"]').attr('content') ?? null; const auction: Auction = { id: m[1]!, slug: m[2]!, title: '', endedOn, lotCount: null, ended: Boolean(endedOn) }; const payload: PagePayload = { kind: 'lot_page', url, auction, page: 0, totalPages: null, lots: [{ itemId: m[3]!, url, title, lotNo, soldText, priceGbp: price ? Number(price.replace(/,/g, '')) : null, image }] }; return [{ url, externalId: `lot:${m[3]}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const saleDate = p.auction.endedOn ? new Date(p.auction.endedOn) : null; if (!saleDate) return []; const out: NormalizedRecord[] = []; for (const lot of p.lots) { if (!lot.priceGbp || lot.priceGbp <= 0) continue; const f = whiskyFacts(lot.title); const attributes = AssetAttributesSchema.parse({ categorySlug: f.categorySlug, brand: f.brand, name: lot.title, year: f.vintage, size: f.size, country: f.categorySlug === 'whisky' && /scotch|islay|speyside|highland|campbeltown|lowland/i.test(lot.title) ? 'GB' : null, identifiers: { swa_item: lot.itemId, ...(lot.lotNo ? { swa_lot: lot.lotNo } : {}) }, metadata: { age_statement: f.age, auction_id: p.auction.id, auction_title: p.auction.title, sold_text: lot.soldText }, }); out.push( NormalizedSaleSchema.parse({ kind: 'sale', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: lot.url, externalId: lot.itemId, rawTitle: lot.title, imageUrls: lot.image ? [lot.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: PARSER_VERSION, saleType: 'auction', saleDate, price: lot.priceGbp, currency: 'GBP', buyerPremiumIncluded: false, quantity: 1, isBundle: /\b(x\s?\d|\d+\s?x\s|lot of|set of|\d+\s?bottles)\b/i.test(lot.title), location: 'Glasgow, United Kingdom', auctionHouse: 'Scotch Whisky Auctions', lotNumber: lot.lotNo, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new ScotchWhiskyAuctionsConnector(meta); }