import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared'; import { whiskyFacts } from '../scotch-whisky-auctions/index.js'; /** * Just Whisky past auctions — public JSON API behind the Past Auctions page. One raw record per API page * (compact lots); one sale per lot whose reserve was met (hammer price). */ const BASE = 'https://www.just-whisky.co.uk'; const PARSER_VERSION = '1.0.0'; export const LotSchema = z.object({ id: z.number(), slug: z.string(), title: z.string(), subtitle: z.string().nullable(), reserveMet: z.boolean(), hammerPrice: z.number().nullable(), currentBid: z.number().nullable(), isGroupLot: z.boolean(), auctionId: z.number().nullable(), auctionEnd: z.string().nullable(), strength: z.string().nullable(), size: z.string().nullable(), distillery: z.string().nullable(), bottler: z.string().nullable(), region: z.string().nullable(), estimatedValue: z.number().nullable(), image: z.string().nullable(), }); export type Lot = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), url: z.string(), page: z.number(), count: z.number().nullable(), lots: z.array(LotSchema) }); interface ApiLot { id: number; slug: string; reserve_met?: boolean; hammer_price?: string | null; is_group_lot?: boolean; custom_title?: string | null; custom_subtitle?: string | null; bid_stats?: { current_bid?: number | null }; seller_sheet?: { auction?: { id?: number; end_date?: string } }; item?: { title?: string; subtitle?: string; strength?: { name?: string } | null; size?: { name?: string } | null; distillery?: { name?: string } | string | null; bottler?: { name?: string } | string | null; region?: { name?: string } | string | null; estimated_value?: string | null; photo?: { file?: string } | null }; photos?: Array<{ file?: string }>; } const name = (v: { name?: string } | string | null | undefined): string | null => (typeof v === 'string' ? v : v?.name ?? null) || null; export function trimLot(l: ApiLot): Lot | null { const title = (l.custom_title || l.item?.title || '').trim(); if (!l.id || !l.slug || !title) return null; const hp = l.hammer_price ? Number(l.hammer_price) : null; return LotSchema.parse({ id: l.id, slug: l.slug, title, subtitle: (l.custom_subtitle || l.item?.subtitle || null)?.trim() || null, reserveMet: Boolean(l.reserve_met), hammerPrice: hp && Number.isFinite(hp) ? hp : null, currentBid: typeof l.bid_stats?.current_bid === 'number' ? l.bid_stats.current_bid : null, isGroupLot: Boolean(l.is_group_lot), auctionId: l.seller_sheet?.auction?.id ?? null, auctionEnd: l.seller_sheet?.auction?.end_date ?? null, strength: name(l.item?.strength), size: name(l.item?.size), distillery: name(l.item?.distillery), bottler: name(l.item?.bottler), region: name(l.item?.region), estimatedValue: l.item?.estimated_value ? Number(l.item.estimated_value) || null : null, image: l.photos?.[0]?.file ?? l.item?.photo?.file ?? null, }); } const ddmmyyyy = (d: Date) => `${String(d.getUTCDate()).padStart(2, '0')}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${d.getUTCFullYear()}`; export function lotsUrl(from: Date, to: Date, page: number, pageSize: number): string { return `${BASE}/api/lots/?min_end_date=${encodeURIComponent(ddmmyyyy(from))}&max_end_date=${encodeURIComponent(ddmmyyyy(to))}&ordering=-price&page_size=${pageSize}&page=${page}`; } export class JustWhiskyConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/www\.just-whisky\.co\.uk\/lot\/[a-z0-9-]+/i]; /** Auction windows: [start-1d, end+1d] for each completed auction (newest first). */ private async windows(ctx: CrawlContext): Promise> { const res = await ctx.fetch(`${BASE}/api/auctions/?page_size=200`, { engines: ['api'], minQuality: 0 }); const data = (res.json as { data?: { results?: Array<{ id: number; name: string; start_date: string; end_date: string; is_published: boolean }> } } | null)?.data?.results ?? []; const now = Date.now(); return data .filter((a) => a.is_published && new Date(a.end_date).getTime() < now && new Date(a.end_date).getUTCFullYear() >= 2013) .map((a) => ({ id: a.id, name: a.name, from: new Date(new Date(a.start_date).getTime() - 86_400_000), to: new Date(new Date(a.end_date).getTime() + 86_400_000) })) .sort((a, b) => b.to.getTime() - a.to.getTime()); } async *crawl(ctx: CrawlContext): AsyncIterable { const pageSize = Number(this.meta.config.pageSize ?? 200); const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 8); const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2); const backfill = ctx.options.mode === 'backfill'; const done = new Set((ctx.options.cursor?.done as number[] | undefined) ?? []); const all = await this.windows(ctx); if (!all.length) { ctx.anomaly('empty_page', 'no auctions from /api/auctions/'); return; } const todo = (backfill ? [...all].reverse() : all).filter((w) => !done.has(w.id)).slice(0, auctionsPerRun); let pages = 0; let count = 0; for (const w of todo) { let page = 1; while (pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, count)) { const url = lotsUrl(w.from, w.to, page, pageSize); await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], expect: ['title', 'price', 'date', 'status'], parse: (r) => { const first = (r.json as { data?: { results?: ApiLot[] } } | null)?.data?.results?.[0]; return first ? { title: first.item?.title ?? first.custom_title, price: first.hammer_price ?? first.bid_stats?.current_bid, date: first.seller_sheet?.auction?.end_date, status: first.reserve_met } : null; }, }); pages++; const data = (res.json as { data?: { results?: ApiLot[]; count?: number; total_pages?: number } } | null)?.data; if (!res.success || !data?.results) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const lots = data.results.map(trimLot).filter((x): x is Lot => Boolean(x)); if (!lots.length) break; count++; yield { url, externalId: `auction:${w.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, page, count: data.count ?? null, lots }, fetchedAt: res.fetchedAt }; if (!data.total_pages || page >= data.total_pages) { done.add(w.id); break; } page++; } await ctx.setCursor({ done: [...done], updatedAt: new Date().toISOString() }); } } async lookup(url: string, ctx: CrawlContext): Promise { const slug = url.match(/\/lot\/([a-z0-9-]+)/i)?.[1]; const id = slug?.match(/(\d+)$/)?.[1]; if (!id) return []; const res = await ctx.fetch(`${BASE}/api/lots/${id}/`, { engines: ['api'], minQuality: 0 }); const data = (res.json as { data?: ApiLot } | null)?.data; const lot = data ? trimLot(data) : null; if (!res.success || !lot) return []; return [{ url, externalId: `lot:${id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, page: 0, count: 1, lots: [lot] }, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const lot of p.lots) { const price = lot.hammerPrice ?? (lot.reserveMet ? lot.currentBid : null); if (!lot.reserveMet || !price || price <= 0 || !lot.auctionEnd) continue; const saleDate = new Date(lot.auctionEnd.endsWith('Z') ? lot.auctionEnd : `${lot.auctionEnd}Z`); if (Number.isNaN(saleDate.getTime())) continue; const fullTitle = lot.subtitle ? `${lot.title} ${lot.subtitle}` : lot.title; const f = whiskyFacts(fullTitle); const attributes = AssetAttributesSchema.parse({ categorySlug: f.categorySlug, brand: lot.distillery ?? f.brand, name: fullTitle, year: f.vintage, size: lot.size && /\d/.test(lot.size) ? lot.size.replace(/\s+/g, '') : f.size, country: /scotch|islay|speyside|highland|campbeltown|lowland|scotland/i.test(`${fullTitle} ${lot.region ?? ''}`) ? 'GB' : null, identifiers: { justwhisky_lot: String(lot.id) }, metadata: { age_statement: f.age, strength: lot.strength, bottler: lot.bottler, region: lot.region, estimated_value_gbp: lot.estimatedValue, auction_id: lot.auctionId, group_lot: lot.isGroupLot }, }); out.push( NormalizedSaleSchema.parse({ kind: 'sale', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${BASE}/lot/${lot.slug}`, externalId: String(lot.id), rawTitle: fullTitle, imageUrls: lot.image ? [lot.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: PARSER_VERSION, saleType: 'auction', saleDate, price, currency: 'GBP', buyerPremiumIncluded: false, quantity: 1, isBundle: lot.isGroupLot || /\bx\s?\d|\(x\d+\)|\bset of\b/i.test(fullTitle), location: 'Scotland, United Kingdom', auctionHouse: 'Just Whisky', lotNumber: null, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): JustWhiskyConnector { return new JustWhiskyConnector(meta); }