TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, NormalizedSaleSchema, parsePrice, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';45/**6 * WineBid — item pages expose a "Recent sales" table (past winning bids for the same wine).7 * One raw record per item page; normalise → one sale per past sale row plus the current listing.8 */910const BASE = 'https://www.winebid.com';11const PARSER_VERSION = '1.0.0';12const PREMIUM_PCT = 17;1314const CategorySchema = z.object({ path: z.string(), label: z.string() });1516export const SaleRowSchema = z.object({ itemId: z.string(), url: z.string(), quantity: z.number(), amountUsd: z.number(), soldOn: z.string() });17export const ItemPayloadSchema = z.object({18 kind: z.literal('item_page'),19 url: z.string(),20 itemId: z.string(),21 title: z.string(),22 category: z.string().nullable(),23 condition: z.string().nullable(),24 currentBid: z.number().nullable(),25 estimate: z.number().nullable(),26 ended: z.boolean().nullable(),27 image: z.string().nullable(),28 recentSales: z.array(SaleRowSchema),29});30export type ItemPayload = z.infer<typeof ItemPayloadSchema>;3132export function parseCategoryPage(text: string): string[] {33 const out = new Set<string>();34 for (const m of text.matchAll(/https:\/\/www\.winebid\.com\/BuyWine\/Item\/(\d+)\/[^)\s"'<>]*/g)) out.add(`${BASE}/BuyWine/Item/${m[1]}/${m[0].split('/').pop()!.replace(/[)"'>]+$/g, '')}`);35 for (const m of text.matchAll(/href="(\/BuyWine\/Item\/(\d+)\/[^"]+)"/g)) out.add(BASE + m[1]!);36 return [...out];37}3839export function parseItemPage(htmlText: string, url: string, category: string | null): ItemPayload | null {40 const $ = H.load(htmlText);41 const itemId = url.match(/\/BuyWine\/Item\/(\d+)/)?.[1];42 const title = H.text($('h1').first());43 if (!itemId || !title) return null;44 const condition = H.text($('h1').first().nextAll('p').first());45 const recentSales: z.infer<typeof SaleRowSchema>[] = [];46 $('table.recentSales tr').each((_, tr) => {47 const tds = $(tr).find('td');48 if (tds.length < 4) return;49 const a = $(tds[0]).find('a');50 const href = a.attr('href') ?? '';51 const id = href.match(/\/BuyWine\/Item\/(\d+)/)?.[1];52 const qty = Number(H.text($(tds[1])) ?? '');53 const amount = parsePrice(H.text($(tds[2])), 'USD');54 const soldOn = $(tds[3]).attr('title') ?? H.text($(tds[3])) ?? '';55 if (!id || !amount || !soldOn) return;56 recentSales.push({ itemId: id, url: BASE + href, quantity: Number.isFinite(qty) && qty > 0 ? qty : 1, amountUsd: amount.amount, soldOn });57 });58 const bidTxt = H.text($('.currentBid, .bidAmount, #currentBid').first()) ?? htmlText.match(/Current Bid[^$]{0,80}\$([\d,]+(?:\.\d+)?)/i)?.[1] ?? null;59 const currentBid = bidTxt ? (parsePrice(bidTxt, 'USD')?.amount ?? null) : null;60 const estimate = parsePrice(htmlText.match(/Estimate[^$]{0,120}\$([\d,]+(?:\.\d+)?)/i)?.[1] ?? null, 'USD')?.amount ?? null;61 const endedAttr = $('#itemDiagnostics').attr('data-is-ended');62 const image = $('meta[property="og:image"]').attr('content') ?? $('img[src*="/Photo/"]').first().attr('src') ?? null;63 return { kind: 'item_page', url, itemId, title, category, condition: condition ?? null, currentBid, estimate, ended: endedAttr ? endedAttr === 'True' : null, image: image ? (image.startsWith('http') ? image : BASE + image) : null, recentSales };64}6566/** "1983 Château Margaux (1.5L)" → { vintage, producer, size } — nothing guessed. */67export function wineFacts(title: string): { vintage: number | null; name: string; size: string | null } {68 const vintage = title.match(/^(19\d{2}|20\d{2})\b/)?.[1] ?? null;69 const size = title.match(/\((\d+(?:\.\d+)?\s?(?:L|ml|mL))\)/i)?.[1]?.replace(/\s/g, '') ?? null;70 const name = title71 .replace(/^(19\d{2}|20\d{2}|NV)\s+/, '')72 .replace(/\s*\((\d+(?:\.\d+)?\s?(?:L|ml|mL))\)\s*/i, ' ')73 .trim();74 return { vintage: vintage ? Number(vintage) : null, name, size: size ?? '750ml' };75}7677export class WineBidConnector extends BaseConnector {78 readonly version = '1.0.0';79 readonly parserVersion = PARSER_VERSION;80 override readonly urlPatterns = [/winebid\.com\/BuyWine\/Item\/\d+/];81 protected override minIntervalMs = 2000;8283 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {84 const categories = z.array(CategorySchema).parse(this.meta.config.categories ?? []);85 const perRun = Number(this.meta.config.itemsPerRun ?? 24);86 const cursor = (ctx.options.cursor ?? {}) as { seen?: string[] };87 const seen = new Set<string>(cursor.seen ?? []);88 let count = 0;89 let fetched = 0;90 const perCategory = Math.max(2, Math.ceil(perRun / Math.max(1, categories.length)));91 for (const cat of categories) {92 if (ctx.signal?.aborted || fetched >= perRun) break;93 const listUrl = BASE + cat.path;94 const list = await ctx.fetch(listUrl, { engines: ['firecrawl'], timeoutMs: 60_000, minQuality: 0 });95 if (!list.success) {96 ctx.anomaly('category_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);97 continue;98 }99 const urls = parseCategoryPage(`${list.markdown ?? ''}\n${list.html ?? ''}`);100 if (urls.length === 0) {101 ctx.anomaly('empty_category', listUrl);102 continue;103 }104 let done = 0;105 for (const url of urls) {106 if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun || done >= perCategory) break;107 const id = url.match(/\/Item\/(\d+)/)?.[1] ?? url;108 if (seen.has(id) && ctx.options.mode !== 'backfill') continue;109 await this.throttle();110 fetched++;111 const res = await ctx.fetch(url, {112 engines: ['api', 'firecrawl'],113 responseType: 'text',114 expect: ['title', 'price', 'currency', 'date'],115 parse: (r) => {116 const p = r.html ? parseItemPage(r.html, url, cat.label) : null;117 const s = p?.recentSales[0];118 return p ? { title: p.title, price: s?.amountUsd ?? p.currentBid, currency: 'USD', date: s?.soldOn ?? null } : null;119 },120 });121 const payload = res.success && res.html ? parseItemPage(res.html, url, cat.label) : null;122 if (!payload) {123 ctx.anomaly('item_parse_failed', `${url}: ${res.error ?? res.httpStatus}`);124 continue;125 }126 seen.add(id);127 done++;128 count++;129 yield { url, externalId: `item:${payload.itemId}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };130 }131 await ctx.setCursor({ seen: [...seen].slice(-3000) });132 }133 }134135 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {136 const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0 });137 const payload = res.success && res.html ? parseItemPage(res.html, url, null) : null;138 return payload ? [{ url, externalId: `item:${payload.itemId}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : [];139 }140141 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {142 const p = ItemPayloadSchema.parse(raw.payload);143 const f = wineFacts(p.title);144 const categorySlug = /whisk|bourbon|scotch/i.test(p.title) ? 'whisky' : 'wine';145 const attributes = AssetAttributesSchema.parse({146 categorySlug,147 series: p.category,148 name: f.name,149 year: f.vintage,150 size: f.size,151 identifiers: { winebid_wine: `${f.vintage ?? 'NV'}|${f.name.toLowerCase()}|${f.size ?? ''}` },152 metadata: { category_page: p.category },153 });154 const common = {155 connectorId: this.meta.id,156 sourceId: this.meta.sourceId,157 attributes,158 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },159 condition: { condition: null, conditionRaw: p.condition, completeness: null },160 observedAt: raw.fetchedAt,161 confidence: 0.85,162 parserVersion: PARSER_VERSION,163 imageUrls: p.image ? [p.image] : [],164 description: null,165 };166 const out: NormalizedRecord[] = [];167 for (const s of p.recentSales) {168 const saleDate = parseSourceDate(s.soldOn.replace(/\s+\d{1,2}:\d{2}:\d{2}\s*(AM|PM)?\s*\([A-Z]{3,4}\)$/i, ''));169 if (!saleDate) continue;170 out.push(171 NormalizedSaleSchema.parse({172 kind: 'sale',173 ...common,174 sourceUrl: s.url,175 externalId: s.itemId,176 rawTitle: p.title,177 saleType: 'auction',178 saleDate,179 price: s.amountUsd,180 currency: 'USD',181 buyerPremiumIncluded: false,182 quantity: s.quantity,183 isBundle: false,184 location: 'United States',185 auctionHouse: 'WineBid',186 lotNumber: s.itemId,187 attributes: { ...attributes, metadata: { ...attributes.metadata, price_basis: 'per_bottle', buyer_premium_pct: PREMIUM_PCT } },188 }),189 );190 }191 if (p.currentBid || p.estimate) {192 out.push(193 NormalizedListingSchema.parse({194 kind: 'listing',195 ...common,196 sourceUrl: p.url,197 externalId: p.itemId,198 rawTitle: p.title,199 listingType: 'auction',200 price: p.currentBid ?? p.estimate,201 currency: 'USD',202 seller: 'WineBid',203 location: 'United States',204 availability: p.ended === true ? 'ended' : 'available',205 attributes: { ...attributes, metadata: { ...attributes.metadata, estimate_usd: p.estimate, price_is_estimate: p.currentBid === null } },206 }),207 );208 }209 return out;210 }211}212213export default function createConnector(meta: ConnectorMeta) {214 return new WineBidConnector(meta);215}216