TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';4import { whiskyFacts } from '../scotch-whisky-auctions/index.js';56/**7 * Just Whisky past auctions — public JSON API behind the Past Auctions page. One raw record per API page8 * (compact lots); one sale per lot whose reserve was met (hammer price).9 */10const BASE = 'https://www.just-whisky.co.uk';11const PARSER_VERSION = '1.0.0';1213export const LotSchema = z.object({14 id: z.number(),15 slug: z.string(),16 title: z.string(),17 subtitle: z.string().nullable(),18 reserveMet: z.boolean(),19 hammerPrice: z.number().nullable(),20 currentBid: z.number().nullable(),21 isGroupLot: z.boolean(),22 auctionId: z.number().nullable(),23 auctionEnd: z.string().nullable(),24 strength: z.string().nullable(),25 size: z.string().nullable(),26 distillery: z.string().nullable(),27 bottler: z.string().nullable(),28 region: z.string().nullable(),29 estimatedValue: z.number().nullable(),30 image: z.string().nullable(),31});32export type Lot = z.infer<typeof LotSchema>;33export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), url: z.string(), page: z.number(), count: z.number().nullable(), lots: z.array(LotSchema) });3435interface ApiLot {36 id: number;37 slug: string;38 reserve_met?: boolean;39 hammer_price?: string | null;40 is_group_lot?: boolean;41 custom_title?: string | null;42 custom_subtitle?: string | null;43 bid_stats?: { current_bid?: number | null };44 seller_sheet?: { auction?: { id?: number; end_date?: string } };45 item?: { title?: string; subtitle?: string; strength?: { name?: string } | null; size?: { name?: string } | null; distillery?: { name?: string } | string | null; bottler?: { name?: string } | string | null; region?: { name?: string } | string | null; estimated_value?: string | null; photo?: { file?: string } | null };46 photos?: Array<{ file?: string }>;47}48const name = (v: { name?: string } | string | null | undefined): string | null => (typeof v === 'string' ? v : v?.name ?? null) || null;4950export function trimLot(l: ApiLot): Lot | null {51 const title = (l.custom_title || l.item?.title || '').trim();52 if (!l.id || !l.slug || !title) return null;53 const hp = l.hammer_price ? Number(l.hammer_price) : null;54 return LotSchema.parse({55 id: l.id,56 slug: l.slug,57 title,58 subtitle: (l.custom_subtitle || l.item?.subtitle || null)?.trim() || null,59 reserveMet: Boolean(l.reserve_met),60 hammerPrice: hp && Number.isFinite(hp) ? hp : null,61 currentBid: typeof l.bid_stats?.current_bid === 'number' ? l.bid_stats.current_bid : null,62 isGroupLot: Boolean(l.is_group_lot),63 auctionId: l.seller_sheet?.auction?.id ?? null,64 auctionEnd: l.seller_sheet?.auction?.end_date ?? null,65 strength: name(l.item?.strength),66 size: name(l.item?.size),67 distillery: name(l.item?.distillery),68 bottler: name(l.item?.bottler),69 region: name(l.item?.region),70 estimatedValue: l.item?.estimated_value ? Number(l.item.estimated_value) || null : null,71 image: l.photos?.[0]?.file ?? l.item?.photo?.file ?? null,72 });73}7475const ddmmyyyy = (d: Date) => `${String(d.getUTCDate()).padStart(2, '0')}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${d.getUTCFullYear()}`;7677export function lotsUrl(from: Date, to: Date, page: number, pageSize: number): string {78 return `${BASE}/api/lots/?min_end_date=${encodeURIComponent(ddmmyyyy(from))}&max_end_date=${encodeURIComponent(ddmmyyyy(to))}&ordering=-price&page_size=${pageSize}&page=${page}`;79}8081export class JustWhiskyConnector extends BaseConnector {82 readonly version = '1.0.0';83 readonly parserVersion = PARSER_VERSION;84 protected override minIntervalMs = 1500;85 override readonly urlPatterns = [/^https?:\/\/www\.just-whisky\.co\.uk\/lot\/[a-z0-9-]+/i];8687 /** Auction windows: [start-1d, end+1d] for each completed auction (newest first). */88 private async windows(ctx: CrawlContext): Promise<Array<{ id: number; from: Date; to: Date; name: string }>> {89 const res = await ctx.fetch(`${BASE}/api/auctions/?page_size=200`, { engines: ['api'], minQuality: 0 });90 const data = (res.json as { data?: { results?: Array<{ id: number; name: string; start_date: string; end_date: string; is_published: boolean }> } } | null)?.data?.results ?? [];91 const now = Date.now();92 return data93 .filter((a) => a.is_published && new Date(a.end_date).getTime() < now && new Date(a.end_date).getUTCFullYear() >= 2013)94 .map((a) => ({ id: a.id, name: a.name, from: new Date(new Date(a.start_date).getTime() - 86_400_000), to: new Date(new Date(a.end_date).getTime() + 86_400_000) }))95 .sort((a, b) => b.to.getTime() - a.to.getTime());96 }9798 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {99 const pageSize = Number(this.meta.config.pageSize ?? 200);100 const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 8);101 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);102 const backfill = ctx.options.mode === 'backfill';103 const done = new Set<number>((ctx.options.cursor?.done as number[] | undefined) ?? []);104 const all = await this.windows(ctx);105 if (!all.length) {106 ctx.anomaly('empty_page', 'no auctions from /api/auctions/');107 return;108 }109 const todo = (backfill ? [...all].reverse() : all).filter((w) => !done.has(w.id)).slice(0, auctionsPerRun);110 let pages = 0;111 let count = 0;112 for (const w of todo) {113 let page = 1;114 while (pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, count)) {115 const url = lotsUrl(w.from, w.to, page, pageSize);116 await this.throttle();117 const res = await ctx.fetch(url, {118 engines: ['api'],119 expect: ['title', 'price', 'date', 'status'],120 parse: (r) => {121 const first = (r.json as { data?: { results?: ApiLot[] } } | null)?.data?.results?.[0];122 return first ? { title: first.item?.title ?? first.custom_title, price: first.hammer_price ?? first.bid_stats?.current_bid, date: first.seller_sheet?.auction?.end_date, status: first.reserve_met } : null;123 },124 });125 pages++;126 const data = (res.json as { data?: { results?: ApiLot[]; count?: number; total_pages?: number } } | null)?.data;127 if (!res.success || !data?.results) {128 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);129 break;130 }131 const lots = data.results.map(trimLot).filter((x): x is Lot => Boolean(x));132 if (!lots.length) break;133 count++;134 yield { url, externalId: `auction:${w.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, page, count: data.count ?? null, lots }, fetchedAt: res.fetchedAt };135 if (!data.total_pages || page >= data.total_pages) {136 done.add(w.id);137 break;138 }139 page++;140 }141 await ctx.setCursor({ done: [...done], updatedAt: new Date().toISOString() });142 }143 }144145 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {146 const slug = url.match(/\/lot\/([a-z0-9-]+)/i)?.[1];147 const id = slug?.match(/(\d+)$/)?.[1];148 if (!id) return [];149 const res = await ctx.fetch(`${BASE}/api/lots/${id}/`, { engines: ['api'], minQuality: 0 });150 const data = (res.json as { data?: ApiLot } | null)?.data;151 const lot = data ? trimLot(data) : null;152 if (!res.success || !lot) return [];153 return [{ url, externalId: `lot:${id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, page: 0, count: 1, lots: [lot] }, fetchedAt: res.fetchedAt }];154 }155156 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {157 const p = PagePayloadSchema.parse(raw.payload);158 const out: NormalizedRecord[] = [];159 for (const lot of p.lots) {160 const price = lot.hammerPrice ?? (lot.reserveMet ? lot.currentBid : null);161 if (!lot.reserveMet || !price || price <= 0 || !lot.auctionEnd) continue;162 const saleDate = new Date(lot.auctionEnd.endsWith('Z') ? lot.auctionEnd : `${lot.auctionEnd}Z`);163 if (Number.isNaN(saleDate.getTime())) continue;164 const fullTitle = lot.subtitle ? `${lot.title} ${lot.subtitle}` : lot.title;165 const f = whiskyFacts(fullTitle);166 const attributes = AssetAttributesSchema.parse({167 categorySlug: f.categorySlug,168 brand: lot.distillery ?? f.brand,169 name: fullTitle,170 year: f.vintage,171 size: lot.size && /\d/.test(lot.size) ? lot.size.replace(/\s+/g, '') : f.size,172 country: /scotch|islay|speyside|highland|campbeltown|lowland|scotland/i.test(`${fullTitle} ${lot.region ?? ''}`) ? 'GB' : null,173 identifiers: { justwhisky_lot: String(lot.id) },174 metadata: { age_statement: f.age, strength: lot.strength, bottler: lot.bottler, region: lot.region, estimated_value_gbp: lot.estimatedValue, auction_id: lot.auctionId, group_lot: lot.isGroupLot },175 });176 out.push(177 NormalizedSaleSchema.parse({178 kind: 'sale',179 connectorId: this.meta.id,180 sourceId: this.meta.sourceId,181 sourceUrl: `${BASE}/lot/${lot.slug}`,182 externalId: String(lot.id),183 rawTitle: fullTitle,184 imageUrls: lot.image ? [lot.image] : [],185 attributes,186 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },187 condition: { condition: null, conditionRaw: null, completeness: null },188 observedAt: raw.fetchedAt,189 confidence: 0.9,190 parserVersion: PARSER_VERSION,191 saleType: 'auction',192 saleDate,193 price,194 currency: 'GBP',195 buyerPremiumIncluded: false,196 quantity: 1,197 isBundle: lot.isGroupLot || /\bx\s?\d|\(x\d+\)|\bset of\b/i.test(fullTitle),198 location: 'Scotland, United Kingdom',199 auctionHouse: 'Just Whisky',200 lotNumber: null,201 }),202 );203 }204 return out;205 }206}207208export default function createConnector(meta: ConnectorMeta): JustWhiskyConnector {209 return new JustWhiskyConnector(meta);210}211