import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, NormalizedSaleSchema, parsePrice, parseSourceDate, type NormalizedRecord } from '@rareindex/shared'; /** * WineBid — item pages expose a "Recent sales" table (past winning bids for the same wine). * One raw record per item page; normalise → one sale per past sale row plus the current listing. */ const BASE = 'https://www.winebid.com'; const PARSER_VERSION = '1.0.0'; const PREMIUM_PCT = 17; const CategorySchema = z.object({ path: z.string(), label: z.string() }); export const SaleRowSchema = z.object({ itemId: z.string(), url: z.string(), quantity: z.number(), amountUsd: z.number(), soldOn: z.string() }); export const ItemPayloadSchema = z.object({ kind: z.literal('item_page'), url: z.string(), itemId: z.string(), title: z.string(), category: z.string().nullable(), condition: z.string().nullable(), currentBid: z.number().nullable(), estimate: z.number().nullable(), ended: z.boolean().nullable(), image: z.string().nullable(), recentSales: z.array(SaleRowSchema), }); export type ItemPayload = z.infer; export function parseCategoryPage(text: string): string[] { const out = new Set(); for (const m of text.matchAll(/https:\/\/www\.winebid\.com\/BuyWine\/Item\/(\d+)\/[^)\s"'<>]*/g)) out.add(`${BASE}/BuyWine/Item/${m[1]}/${m[0].split('/').pop()!.replace(/[)"'>]+$/g, '')}`); for (const m of text.matchAll(/href="(\/BuyWine\/Item\/(\d+)\/[^"]+)"/g)) out.add(BASE + m[1]!); return [...out]; } export function parseItemPage(htmlText: string, url: string, category: string | null): ItemPayload | null { const $ = H.load(htmlText); const itemId = url.match(/\/BuyWine\/Item\/(\d+)/)?.[1]; const title = H.text($('h1').first()); if (!itemId || !title) return null; const condition = H.text($('h1').first().nextAll('p').first()); const recentSales: z.infer[] = []; $('table.recentSales tr').each((_, tr) => { const tds = $(tr).find('td'); if (tds.length < 4) return; const a = $(tds[0]).find('a'); const href = a.attr('href') ?? ''; const id = href.match(/\/BuyWine\/Item\/(\d+)/)?.[1]; const qty = Number(H.text($(tds[1])) ?? ''); const amount = parsePrice(H.text($(tds[2])), 'USD'); const soldOn = $(tds[3]).attr('title') ?? H.text($(tds[3])) ?? ''; if (!id || !amount || !soldOn) return; recentSales.push({ itemId: id, url: BASE + href, quantity: Number.isFinite(qty) && qty > 0 ? qty : 1, amountUsd: amount.amount, soldOn }); }); const bidTxt = H.text($('.currentBid, .bidAmount, #currentBid').first()) ?? htmlText.match(/Current Bid[^$]{0,80}\$([\d,]+(?:\.\d+)?)/i)?.[1] ?? null; const currentBid = bidTxt ? (parsePrice(bidTxt, 'USD')?.amount ?? null) : null; const estimate = parsePrice(htmlText.match(/Estimate[^$]{0,120}\$([\d,]+(?:\.\d+)?)/i)?.[1] ?? null, 'USD')?.amount ?? null; const endedAttr = $('#itemDiagnostics').attr('data-is-ended'); const image = $('meta[property="og:image"]').attr('content') ?? $('img[src*="/Photo/"]').first().attr('src') ?? null; return { kind: 'item_page', url, itemId, title, category, condition: condition ?? null, currentBid, estimate, ended: endedAttr ? endedAttr === 'True' : null, image: image ? (image.startsWith('http') ? image : BASE + image) : null, recentSales }; } /** "1983 Château Margaux (1.5L)" → { vintage, producer, size } — nothing guessed. */ export function wineFacts(title: string): { vintage: number | null; name: string; size: string | null } { const vintage = title.match(/^(19\d{2}|20\d{2})\b/)?.[1] ?? null; const size = title.match(/\((\d+(?:\.\d+)?\s?(?:L|ml|mL))\)/i)?.[1]?.replace(/\s/g, '') ?? null; const name = title .replace(/^(19\d{2}|20\d{2}|NV)\s+/, '') .replace(/\s*\((\d+(?:\.\d+)?\s?(?:L|ml|mL))\)\s*/i, ' ') .trim(); return { vintage: vintage ? Number(vintage) : null, name, size: size ?? '750ml' }; } export class WineBidConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; override readonly urlPatterns = [/winebid\.com\/BuyWine\/Item\/\d+/]; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const categories = z.array(CategorySchema).parse(this.meta.config.categories ?? []); const perRun = Number(this.meta.config.itemsPerRun ?? 24); const cursor = (ctx.options.cursor ?? {}) as { seen?: string[] }; const seen = new Set(cursor.seen ?? []); let count = 0; let fetched = 0; const perCategory = Math.max(2, Math.ceil(perRun / Math.max(1, categories.length))); for (const cat of categories) { if (ctx.signal?.aborted || fetched >= perRun) break; const listUrl = BASE + cat.path; const list = await ctx.fetch(listUrl, { engines: ['firecrawl'], timeoutMs: 60_000, minQuality: 0 }); if (!list.success) { ctx.anomaly('category_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`); continue; } const urls = parseCategoryPage(`${list.markdown ?? ''}\n${list.html ?? ''}`); if (urls.length === 0) { ctx.anomaly('empty_category', listUrl); continue; } let done = 0; for (const url of urls) { if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun || done >= perCategory) break; const id = url.match(/\/Item\/(\d+)/)?.[1] ?? url; if (seen.has(id) && ctx.options.mode !== 'backfill') continue; await this.throttle(); fetched++; const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', expect: ['title', 'price', 'currency', 'date'], parse: (r) => { const p = r.html ? parseItemPage(r.html, url, cat.label) : null; const s = p?.recentSales[0]; return p ? { title: p.title, price: s?.amountUsd ?? p.currentBid, currency: 'USD', date: s?.soldOn ?? null } : null; }, }); const payload = res.success && res.html ? parseItemPage(res.html, url, cat.label) : null; if (!payload) { ctx.anomaly('item_parse_failed', `${url}: ${res.error ?? res.httpStatus}`); continue; } seen.add(id); done++; count++; yield { url, externalId: `item:${payload.itemId}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ seen: [...seen].slice(-3000) }); } } async lookup(url: string, ctx: CrawlContext): Promise { const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0 }); const payload = res.success && res.html ? parseItemPage(res.html, url, null) : null; return payload ? [{ url, externalId: `item:${payload.itemId}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : []; } async normalize(raw: RawRecordLike): Promise { const p = ItemPayloadSchema.parse(raw.payload); const f = wineFacts(p.title); const categorySlug = /whisk|bourbon|scotch/i.test(p.title) ? 'whisky' : 'wine'; const attributes = AssetAttributesSchema.parse({ categorySlug, series: p.category, name: f.name, year: f.vintage, size: f.size, identifiers: { winebid_wine: `${f.vintage ?? 'NV'}|${f.name.toLowerCase()}|${f.size ?? ''}` }, metadata: { category_page: p.category }, }); const common = { connectorId: this.meta.id, sourceId: this.meta.sourceId, attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: p.condition, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, imageUrls: p.image ? [p.image] : [], description: null, }; const out: NormalizedRecord[] = []; for (const s of p.recentSales) { const saleDate = parseSourceDate(s.soldOn.replace(/\s+\d{1,2}:\d{2}:\d{2}\s*(AM|PM)?\s*\([A-Z]{3,4}\)$/i, '')); if (!saleDate) continue; out.push( NormalizedSaleSchema.parse({ kind: 'sale', ...common, sourceUrl: s.url, externalId: s.itemId, rawTitle: p.title, saleType: 'auction', saleDate, price: s.amountUsd, currency: 'USD', buyerPremiumIncluded: false, quantity: s.quantity, isBundle: false, location: 'United States', auctionHouse: 'WineBid', lotNumber: s.itemId, attributes: { ...attributes, metadata: { ...attributes.metadata, price_basis: 'per_bottle', buyer_premium_pct: PREMIUM_PCT } }, }), ); } if (p.currentBid || p.estimate) { out.push( NormalizedListingSchema.parse({ kind: 'listing', ...common, sourceUrl: p.url, externalId: p.itemId, rawTitle: p.title, listingType: 'auction', price: p.currentBid ?? p.estimate, currency: 'USD', seller: 'WineBid', location: 'United States', availability: p.ended === true ? 'ended' : 'available', attributes: { ...attributes, metadata: { ...attributes.metadata, estimate_usd: p.estimate, price_is_estimate: p.currentBid === null } }, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new WineBidConnector(meta); }