TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parseSourceDate, AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';45/**6 * Scotch Whisky Auctions — monthly online whisky auctions (Glasgow) with a public archive of7 * results. One raw record per lot-list page; normalise → one sale per lot with a "Sold for" price.8 */910const BASE = 'https://www.scotchwhiskyauctions.com';11const PARSER_VERSION = '1.0.0';1213export const AuctionSchema = z.object({ id: z.string(), slug: z.string(), title: z.string(), endedOn: z.string().nullable(), lotCount: z.number().nullable(), ended: z.boolean() });14export type Auction = z.infer<typeof AuctionSchema>;1516export const LotSchema = z.object({ itemId: z.string(), url: z.string(), title: z.string(), lotNo: z.string().nullable(), soldText: z.string().nullable(), priceGbp: z.number().nullable(), image: z.string().nullable() });17export const PagePayloadSchema = z.object({ kind: z.literal('lot_page'), url: z.string(), auction: AuctionSchema, page: z.number(), totalPages: z.number().nullable(), lots: z.array(LotSchema) });18export type PagePayload = z.infer<typeof PagePayloadSchema>;1920/** Parse /auctions/ : cards "The 181st Auction · Ended July 12, 2026 · There are 4926 lots in this auction". */21export function parseAuctionList(htmlText: string): Auction[] {22 const $ = H.load(htmlText);23 const out: Auction[] = [];24 $('a.auction').each((_, a) => {25 const href = $(a).attr('href') ?? '';26 const m = href.match(/^\/auctions\/(\d+)-([^/]+)\/?$/);27 if (!m) return;28 const title = H.text($(a).find('h4')) ?? '';29 const status = H.text($(a).find('h5')) ?? '';30 const lots = H.text($(a).find('h6'))?.match(/(\d[\d,]*)\s+lots/i)?.[1] ?? null;31 const ended = /^Ended\b/i.test(status);32 const dateTxt = status.replace(/^(Ended|Ends)\s*/i, '');33 const d = parseSourceDate(dateTxt);34 out.push({ id: m[1]!, slug: m[2]!, title, endedOn: d ? d.toISOString() : null, lotCount: lots ? Number(lots.replace(/,/g, '')) : null, ended });35 });36 return out;37}3839/** Parse an auction lot-list page (20 lots): title, lot number, "Sold for £100 in July 2026", image. */40export function parseLotPage(htmlText: string, url: string, auction: Auction, page: number): PagePayload {41 const $ = H.load(htmlText);42 const totalPages = Number(htmlText.match(/Page \d+ of (\d+)/)?.[1]) || null;43 const lots: z.infer<typeof LotSchema>[] = [];44 $('a.lot').each((_, a) => {45 const href = $(a).attr('href') ?? '';46 const m = href.match(/\/auctions\/\d+-[^/]+\/(\d+)-[^/]*\/?$/);47 if (!m) return;48 const title = H.text($(a).find('h4'));49 if (!title) return;50 const lotNo = H.text($(a).find('h6'))?.replace(/^Lot no\s*/i, '') ?? null;51 const soldText = H.text($(a).find('p.sold')) ?? null;52 const price = soldText?.match(/Sold for £([\d,]+(?:\.\d+)?)/i)?.[1] ?? null;53 const bg = $(a).find('.aucimg').attr('style') ?? '';54 const image = bg.match(/url\('([^']+)'\)/)?.[1] ?? null;55 lots.push({ itemId: m[1]!, url: BASE + href.replace(/\/?$/, '/'), title, lotNo, soldText, priceGbp: price ? Number(price.replace(/,/g, '')) : null, image });56 });57 return { kind: 'lot_page', url, auction, page, totalPages, lots };58}5960const SIZE_RE = /(\d+(?:\.\d+)?)\s?(cl|ml|l|litre|liter)s?\b/i;61const AGE_RE = /(\d{1,2})\s*[- ]?\s*(?:year|yo\b|y\.?o\.?)/i;62const VINTAGE_RE = /\b(19[2-9]\d|20[0-2]\d)\b(?!\s*(?:year|yo))/i;6364export interface WhiskyFacts {65 brand: string | null;66 age: number | null;67 vintage: number | null;68 size: string | null;69 categorySlug: 'whisky' | 'rum' | 'cognac';70}7172/** Heuristic facts from a lot title ("Macallan 1989 18 Year Old Gran Reserva 70cl"). Unknown → null, never guessed. */73export function whiskyFacts(title: string): WhiskyFacts {74 const t = title.replace(/[‘’]/g, "'");75 const categorySlug: WhiskyFacts['categorySlug'] = /\brum\b|\brhum\b/i.test(t) ? 'rum' : /\bcognac\b|\barmagnac\b/i.test(t) ? 'cognac' : 'whisky';76 const age = t.match(AGE_RE)?.[1] ? Number(t.match(AGE_RE)![1]) : null;77 const vintage = t.match(VINTAGE_RE)?.[1] ? Number(t.match(VINTAGE_RE)![1]) : null;78 const sizeM = t.match(SIZE_RE);79 const size = sizeM ? `${sizeM[1]}${sizeM[2]!.toLowerCase().startsWith('l') ? 'L' : sizeM[2]!.toLowerCase()}` : null;80 // Brand = leading words before the first digit / age / "Year" / "-" ; keep 1–4 words81 const lead = t.split(/\s+(?=\d)|\s+-\s+|\s+\(/)[0] ?? t;82 const words = lead.replace(/^['"]|['"]$/g, '').trim().split(/\s+/).filter(Boolean);83 const brand = words.length ? words.slice(0, Math.min(words.length, 4)).join(' ') : null;84 return { brand: brand && /[a-z]/i.test(brand) ? brand : null, age, vintage, size, categorySlug };85}8687export class ScotchWhiskyAuctionsConnector extends BaseConnector {88 readonly version = '1.0.0';89 readonly parserVersion = PARSER_VERSION;90 override readonly urlPatterns = [/scotchwhiskyauctions\.com\/auctions\/\d+-[^/]+\/\d+-/];91 protected override minIntervalMs = 2000;9293 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {94 const listUrl = String(this.meta.config.auctionsUrl ?? `${BASE}/auctions/`);95 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);96 const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 12);97 const cursor = (ctx.options.cursor ?? {}) as { progress?: Record<string, number>; complete?: string[] };98 const progress: Record<string, number> = { ...(cursor.progress ?? {}) };99 const complete = new Set<string>(cursor.complete ?? []);100101 const list = await ctx.fetch(listUrl, { engines: ['api'], responseType: 'text', minQuality: 0 });102 if (!list.success || !list.html) {103 ctx.anomaly('auction_list_failed', list.error ?? String(list.httpStatus));104 return;105 }106 let auctions = parseAuctionList(list.html).filter((a) => a.ended);107 // newest first; incremental = newest auctions not yet complete, backfill = continue with older ones108 auctions.sort((a, b) => Number(b.id) - Number(a.id));109 if (ctx.options.mode !== 'backfill') auctions = auctions.slice(0, 6);110 let count = 0;111 let pagesFetched = 0;112 let auctionsTouched = 0;113 for (const auction of auctions) {114 if (complete.has(auction.id)) continue;115 if (auctionsTouched >= auctionsPerRun) break;116 auctionsTouched++;117 let page = (progress[auction.id] ?? 0) + 1;118 for (;;) {119 if (ctx.signal?.aborted || this.reached(ctx, count) || pagesFetched >= pagesPerRun) return void (await ctx.setCursor({ progress, complete: [...complete] }));120 const url = `${BASE}/auctions/${auction.id}-${auction.slug}/${page > 1 ? `?page=${page}` : ''}`;121 await this.throttle();122 const res = await ctx.fetch(url, {123 engines: ['api'],124 responseType: 'text',125 expect: ['title', 'price', 'currency', 'status'],126 parse: (r) => {127 const p = r.html ? parseLotPage(r.html, url, auction, page) : null;128 const sold = p?.lots.find((l) => l.priceGbp);129 return p && p.lots.length ? { title: p.lots[0]!.title, price: sold?.priceGbp ?? null, currency: sold ? 'GBP' : null, status: sold ? 'sold' : null } : null;130 },131 });132 pagesFetched++;133 const payload = res.success && res.html ? parseLotPage(res.html, url, auction, page) : null;134 if (!payload || payload.lots.length === 0) {135 ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);136 complete.add(auction.id);137 break;138 }139 count++;140 yield { url, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };141 progress[auction.id] = page;142 if (payload.totalPages && page >= payload.totalPages) {143 complete.add(auction.id);144 break;145 }146 page++;147 }148 await ctx.setCursor({ progress, complete: [...complete] });149 }150 }151152 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {153 const m = url.match(/\/auctions\/(\d+)-([^/]+)\/(\d+)-/);154 if (!m) return [];155 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 });156 if (!res.success || !res.html) return [];157 const $ = H.load(res.html);158 const title = H.text($('h1').first()) ?? H.text($('h4').first());159 const soldText = res.html.match(/Sold for £[\d,]+(?:\.\d+)?[^<]*/)?.[0] ?? null;160 const lotNo = res.html.match(/Lot no\s*([\d-]+)/i)?.[1] ?? null;161 const ended = res.html.match(/Ended\s+([A-Z][a-z]+ \d{1,2}, \d{4})/)?.[1] ?? null;162 const endedOn = ended ? parseSourceDate(ended)?.toISOString() ?? null : null;163 if (!title) return [];164 const price = soldText?.match(/£([\d,]+(?:\.\d+)?)/)?.[1] ?? null;165 const image = $('meta[property="og:image"]').attr('content') ?? null;166 const auction: Auction = { id: m[1]!, slug: m[2]!, title: '', endedOn, lotCount: null, ended: Boolean(endedOn) };167 const payload: PagePayload = { kind: 'lot_page', url, auction, page: 0, totalPages: null, lots: [{ itemId: m[3]!, url, title, lotNo, soldText, priceGbp: price ? Number(price.replace(/,/g, '')) : null, image }] };168 return [{ url, externalId: `lot:${m[3]}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];169 }170171 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {172 const p = PagePayloadSchema.parse(raw.payload);173 const saleDate = p.auction.endedOn ? new Date(p.auction.endedOn) : null;174 if (!saleDate) return [];175 const out: NormalizedRecord[] = [];176 for (const lot of p.lots) {177 if (!lot.priceGbp || lot.priceGbp <= 0) continue;178 const f = whiskyFacts(lot.title);179 const attributes = AssetAttributesSchema.parse({180 categorySlug: f.categorySlug,181 brand: f.brand,182 name: lot.title,183 year: f.vintage,184 size: f.size,185 country: f.categorySlug === 'whisky' && /scotch|islay|speyside|highland|campbeltown|lowland/i.test(lot.title) ? 'GB' : null,186 identifiers: { swa_item: lot.itemId, ...(lot.lotNo ? { swa_lot: lot.lotNo } : {}) },187 metadata: { age_statement: f.age, auction_id: p.auction.id, auction_title: p.auction.title, sold_text: lot.soldText },188 });189 out.push(190 NormalizedSaleSchema.parse({191 kind: 'sale',192 connectorId: this.meta.id,193 sourceId: this.meta.sourceId,194 sourceUrl: lot.url,195 externalId: lot.itemId,196 rawTitle: lot.title,197 imageUrls: lot.image ? [lot.image] : [],198 attributes,199 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },200 condition: { condition: null, conditionRaw: null, completeness: null },201 observedAt: raw.fetchedAt,202 confidence: 0.9,203 parserVersion: PARSER_VERSION,204 saleType: 'auction',205 saleDate,206 price: lot.priceGbp,207 currency: 'GBP',208 buyerPremiumIncluded: false,209 quantity: 1,210 isBundle: /\b(x\s?\d|\d+\s?x\s|lot of|set of|\d+\s?bottles)\b/i.test(lot.title),211 location: 'Glasgow, United Kingdom',212 auctionHouse: 'Scotch Whisky Auctions',213 lotNumber: lot.lotNo,214 }),215 );216 }217 return out;218 }219}220221export default function createConnector(meta: ConnectorMeta) {222 return new ScotchWhiskyAuctionsConnector(meta);223}224