spb/rareindex

1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';4import { whiskyFacts } from '../../api/scotch-whisky-auctions/index.js';5import { dateDMY } from '../_carlib/index.js';67/**8 * Whisky Hammer previous auctions. Auction list is plain HTML; lot pages come through Firecrawl markdown.9 * One raw record per lot page (compact lots); one sale per lot with a "Sold dd/mm/yyyy £X" line.10 */11const BASE = 'https://www.whiskyhammer.com';12const PARSER_VERSION = '1.0.0';1314export const LotSchema = z.object({ itemId: z.string(), url: z.string(), title: z.string(), soldOn: z.string().nullable(), priceGbp: z.number().nullable(), image: z.string().nullable(), warehouse: z.string().nullable() });15export type Lot = z.infer<typeof LotSchema>;16export const PagePayloadSchema = z.object({ kind: z.literal('lot_page'), url: z.string(), auctionId: z.string(), page: z.number(), totalPages: z.number().nullable(), totalItems: z.number().nullable(), lots: z.array(LotSchema) });1718/** /previous-auctions → auction ids (auc-129 …), newest first as listed. */19export function parseAuctionList(htmlText: string): string[] {20  const $ = H.load(htmlText);21  const ids: string[] = [];22  $('a[href*="/auction/past/auc-"]').each((_, a) => {23    const id = ($(a).attr('href') ?? '').match(/auc-(\d+)/)?.[1];24    if (id && !ids.includes(id)) ids.push(id);25  });26  return ids.sort((a, b) => Number(b) - Number(a));27}2829/**30 * Parse a Firecrawl markdown lot page. Blocks look like:31 *   [![Title](img)](/item/239555/…) … Lot #239555 … [Title](/item/239555/…) … Sold 22/02/2026£9,100.00€… … [View Lot](…)32 */33export function parseLotMarkdown(md: string): { lots: Lot[]; totalPages: number | null; totalItems: number | null } {34  const lots: Lot[] = [];35  const seen = new Set<string>();36  const totalItems = Number(md.match(/(\d[\d,]*)\s+Items/)?.[1]?.replace(/,/g, '')) || null;37  const pageNums = [...md.matchAll(/\?page=(\d+)\)/g)].map((m) => Number(m[1]));38  const totalPages = pageNums.length ? Math.max(...pageNums) : null;39  const re = /Lot #(\d+)\s*\n+\s*\[([^\]]+)\]\((https:\/\/www\.whiskyhammer\.com\/item\/\1\/[^)\s]+)\)\s*\n+\s*(Sold\s+(\d{2}\/\d{2}\/\d{4})\s*£([\d,]+(?:\.\d+)?))?/g;40  let m: RegExpExecArray | null;41  while ((m = re.exec(md))) {42    const itemId = m[1]!;43    if (seen.has(itemId)) continue;44    seen.add(itemId);45    const before = md.slice(Math.max(0, m.index - 1500), m.index);46    const img = [...before.matchAll(/!\[[^\]]*\]\((https:\/\/www\.whiskyhammer\.com\/uploads\/images\/products\/[^)\s]+)\)/g)].at(-1)?.[1] ?? null;47    const warehouse = /EU warehouse/i.test(before.slice(-600)) ? 'NL' : 'GB';48    lots.push({ itemId, url: m[3]!, title: m[2]!.replace(/\\/g, '').trim(), soldOn: m[5] ?? null, priceGbp: m[6] ? Number(m[6].replace(/,/g, '')) : null, image: img, warehouse });49  }50  return { lots, totalPages, totalItems };51}5253export class WhiskyHammerConnector extends BaseConnector {54  readonly version = '1.0.0';55  readonly parserVersion = PARSER_VERSION;56  protected override minIntervalMs = 2000;5758  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {59    const perRun = Number(this.meta.config.auctionsPerRun ?? 1);60    const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 6);61    const progress = { ...((ctx.options.cursor?.progress as Record<string, number> | undefined) ?? {}) };62    const complete = new Set<string>((ctx.options.cursor?.complete as string[] | undefined) ?? []);63    const list = await ctx.fetch(`${BASE}/previous-auctions`, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0 });64    const ids = list.success && list.html ? parseAuctionList(list.html) : [];65    if (!ids.length) {66      ctx.anomaly('page_fetch_failed', `auction list: ${list.error ?? list.httpStatus}`);67      return;68    }69    // The newest entry can still be closing (page not yet published); tolerate a few unreachable auctions per run.70    const candidates = ids.filter((id) => !complete.has(id)).slice(0, perRun + 3);71    let pages = 0;72    let count = 0;73    let started = 0;74    for (const auctionId of candidates) {75      if (started >= perRun) break;76      let page = (progress[auctionId] ?? 0) + 1;77      while (pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, count)) {78        const url = `${BASE}/auction/past/auc-${auctionId}/${page > 1 ? `?page=${page}` : ''}`;79        await this.throttle();80        const res = await ctx.fetch(url, {81          engines: ['firecrawl'],82          timeoutMs: 150_000,83          expect: ['title', 'price', 'currency', 'date'],84          parse: (r) => {85            const p = r.markdown ? parseLotMarkdown(r.markdown) : null;86            const sold = p?.lots.find((l) => l.priceGbp);87            return p && p.lots.length ? { title: p.lots[0]!.title, price: sold?.priceGbp ?? null, currency: sold ? 'GBP' : null, date: sold?.soldOn ?? null } : null;88          },89        });90        pages++;91        const parsed = res.success && res.markdown ? parseLotMarkdown(res.markdown) : null;92        if (!parsed || !parsed.lots.length) {93          ctx.anomaly(parsed ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);94          if (page > 1 || parsed) complete.add(auctionId); // unreachable first page: retry next run95          break;96        }97        if (page === 1) started++;98        count++;99        yield { url, externalId: `auction:${auctionId}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lot_page' as const, url, auctionId, page, totalPages: parsed.totalPages, totalItems: parsed.totalItems, lots: parsed.lots }, fetchedAt: res.fetchedAt };100        progress[auctionId] = page;101        if (parsed.totalPages && page >= parsed.totalPages) {102          complete.add(auctionId);103          break;104        }105        page++;106      }107      await ctx.setCursor({ progress, complete: [...complete] });108    }109  }110111  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {112    const p = PagePayloadSchema.parse(raw.payload);113    const out: NormalizedRecord[] = [];114    for (const lot of p.lots) {115      const saleDate = dateDMY(lot.soldOn);116      if (!lot.priceGbp || lot.priceGbp <= 0 || !saleDate) continue;117      const f = whiskyFacts(lot.title.replace(/\s+-\s+/, ' '));118      const attributes = AssetAttributesSchema.parse({119        categorySlug: f.categorySlug,120        brand: lot.title.split(/\s+-\s+/)[0]?.trim() || f.brand,121        name: lot.title,122        year: f.vintage,123        size: f.size,124        country: /scotch|islay|speyside|highland|campbeltown|lowland/i.test(lot.title) ? 'GB' : null,125        identifiers: { whiskyhammer_item: lot.itemId },126        metadata: { age_statement: f.age, auction_id: p.auctionId, warehouse: lot.warehouse },127      });128      out.push(129        NormalizedSaleSchema.parse({130          kind: 'sale',131          connectorId: this.meta.id,132          sourceId: this.meta.sourceId,133          sourceUrl: lot.url,134          externalId: lot.itemId,135          rawTitle: lot.title,136          imageUrls: lot.image ? [lot.image] : [],137          attributes,138          grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },139          condition: { condition: null, conditionRaw: null, completeness: null },140          observedAt: raw.fetchedAt,141          confidence: 0.9,142          parserVersion: PARSER_VERSION,143          saleType: 'auction',144          saleDate,145          price: lot.priceGbp,146          currency: 'GBP',147          buyerPremiumIncluded: false,148          quantity: 1,149          isBundle: /\bx\s?\d|\(\d+\s?x\)|\bset of\b|\blot of\b/i.test(lot.title),150          location: lot.warehouse === 'NL' ? 'Alphen aan den Rijn, Netherlands' : 'Aberdeenshire, United Kingdom',151          auctionHouse: 'Whisky Hammer',152          lotNumber: lot.itemId,153        }),154      );155    }156    return out;157  }158}159160export default function createConnector(meta: ConnectorMeta): WhiskyHammerConnector {161  return new WhiskyHammerConnector(meta);162}163