TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';4import { dateWords } from '../_carlib/index.js';56/**7 * Hart Davis Hart prices realized (PDF per auction, parsed to markdown tables by Firecrawl).8 * One raw record per PDF (compact rows); one sale per lot with a hammer price.9 */10const PARSER_VERSION = '1.0.0';1112export const RowSchema = z.object({ lot: z.string(), qty: z.number().nullable(), description: z.string(), estimate: z.string().nullable(), hammer: z.number().nullable(), aggregate: z.number().nullable() });13export type Row = z.infer<typeof RowSchema>;14export const PdfPayloadSchema = z.object({ kind: z.literal('results_pdf'), url: z.string(), auctionName: z.string().nullable(), saleDate: z.string().nullable(), rows: z.array(RowSchema) });15export type PdfPayload = z.infer<typeof PdfPayloadSchema>;1617const num = (s: string | undefined): number | null => {18 const v = Number((s ?? '').replace(/[,$\s]/g, ''));19 return Number.isFinite(v) && v > 0 ? v : null;20};2122/** Archive page markdown → [{ pdfUrl, title, dateText }] newest first. */23export function parseArchive(md: string): Array<{ pdfUrl: string; title: string | null; dateText: string | null }> {24 const out: Array<{ pdfUrl: string; title: string | null; dateText: string | null }> = [];25 const seen = new Set<string>();26 for (const m of md.matchAll(/\[View auction results\]\((https:\/\/hdhauctions\.com\/wp-content\/uploads\/[^)\s]+\.pdf)\)/gi)) {27 const url = m[1]!;28 if (seen.has(url)) continue;29 seen.add(url);30 const before = md.slice(Math.max(0, m.index! - 1200), m.index);31 const title = [...before.matchAll(/\*\*([^*\n]{6,120})\*\*/g)].map((x) => x[1]!.trim()).find((t) => !/SOLD|\$|estimate/i.test(t)) ?? null;32 const dateText = [...before.matchAll(/([A-Z][a-z]+ \d{1,2}(?:\s*[-–&]\s*\d{1,2})?,? \d{4})/g)].at(-1)?.[1] ?? null;33 out.push({ pdfUrl: url, title, dateText });34 }35 return out;36}3738/** PDF markdown → header facts + table rows (Lot | Qty | Description | Estimate | Hammer | Aggregate). */39export function parseResultsPdf(md: string): { auctionName: string | null; saleDate: string | null; rows: Row[] } {40 const rows: Row[] = [];41 for (const line of md.split('\n')) {42 const m = line.match(/^\|\s*(\d+[A-Z]?)\s*\|\s*(\d*)\s*\|\s*(.+?)\s*\|\s*([\d,]*\s*-?\s*[\d,]*)\s*\|\s*([\d,]*)\s*\|\s*([\d,.]*)\s*\|/);43 if (!m) continue;44 const description = m[3]!.replace(/\\/g, '').trim();45 if (!description || /^Description$/i.test(description)) continue;46 rows.push({ lot: m[1]!, qty: m[2] ? Number(m[2]) : null, description, estimate: m[4]?.trim() || null, hammer: num(m[5]), aggregate: num(m[6]) });47 }48 const header = md.slice(0, 4000);49 const dateText = header.match(/([A-Z][a-z]+ \d{1,2}(?:\s*[-–]\s*\d{1,2})?,? \d{4})/)?.[1] ?? null;50 const saleDate = dateText ? dateWords(dateText)?.toISOString() ?? null : null;51 const auctionName = header.match(/^#+\s*(.+)$/m)?.[1]?.replace(/Aggregate:.*$/, '').trim() || null;52 return { auctionName, saleDate, rows };53}5455const SIZE_RE = /\((\d+(?:\.\d+)?\s?(?:ml|L|l|cl))\)/;56/** "2010 Château Ducru-Beaucaillou" → vintage, producer/name, bottle size. */57export function wineFacts(description: string): { vintage: number | null; name: string; size: string; producer: string | null; categorySlug: 'wine' | 'whisky' | 'cognac' } {58 const vintage = Number(description.match(/^((?:18|19|20)\d{2})\b/)?.[1]) || null;59 const sizeM = description.match(SIZE_RE);60 const size = sizeM ? sizeM[1]!.replace(/\s/g, '') : '750ml';61 const name = description.replace(/^(?:18|19|20)\d{2}\s+/, '').replace(SIZE_RE, '').replace(/\s+/g, ' ').trim();62 const producer = name.includes(',') ? name.split(',').at(-1)!.trim() : name.split(/\s+/).slice(0, 3).join(' ');63 const categorySlug = /whisky|whiskey|scotch|bourbon/i.test(description) ? 'whisky' : /cognac|armagnac/i.test(description) ? 'cognac' : 'wine';64 return { vintage, name, size, producer: producer || null, categorySlug };65}6667export class HdhWineConnector extends BaseConnector {68 readonly version = '1.0.0';69 readonly parserVersion = PARSER_VERSION;70 protected override minIntervalMs = 3000;7172 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {73 const archiveUrl = String(this.meta.config.archiveUrl ?? 'https://hdhauctions.com/auction-archives/');74 const perRun = Number(this.meta.config.pdfsPerRun ?? 1);75 const done = new Set<string>((ctx.options.cursor?.done as string[] | undefined) ?? []);76 const archive = await ctx.fetch(archiveUrl, { engines: ['firecrawl'], expect: ['title'], parse: (r) => (r.markdown ? { title: parseArchive(r.markdown)[0]?.pdfUrl ?? null } : null) });77 const entries = archive.success && archive.markdown ? parseArchive(archive.markdown) : [];78 if (!entries.length) {79 ctx.anomaly('page_fetch_failed', `archive: ${archive.error ?? archive.httpStatus}`);80 return;81 }82 const order = ctx.options.mode === 'backfill' ? [...entries].reverse() : entries;83 let count = 0;84 for (const e of order.filter((x) => !done.has(x.pdfUrl)).slice(0, perRun)) {85 if (ctx.signal?.aborted || this.reached(ctx, count)) break;86 await this.throttle();87 const res = await ctx.fetch(e.pdfUrl, { engines: ['firecrawl'], timeoutMs: 180_000, expect: ['title', 'price', 'date'], parse: (r) => {88 const p = r.markdown ? parseResultsPdf(r.markdown) : null;89 const sold = p?.rows.find((x) => x.hammer);90 return p && p.rows.length ? { title: sold?.description ?? p.rows[0]!.description, price: sold?.hammer ?? null, date: p.saleDate ?? e.dateText } : null;91 } });92 const parsed = res.success && res.markdown ? parseResultsPdf(res.markdown) : null;93 if (!parsed || !parsed.rows.length) {94 ctx.anomaly(parsed ? 'empty_page' : 'page_fetch_failed', `${e.pdfUrl}: ${res.error ?? res.httpStatus}`);95 continue;96 }97 const saleDate = parsed.saleDate ?? (e.dateText ? dateWords(e.dateText)?.toISOString() ?? null : null);98 count++;99 yield { url: e.pdfUrl, externalId: `pdf:${e.pdfUrl.split('/').pop()}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'results_pdf' as const, url: e.pdfUrl, auctionName: e.title ?? parsed.auctionName, saleDate, rows: parsed.rows }, fetchedAt: res.fetchedAt };100 done.add(e.pdfUrl);101 await ctx.setCursor({ done: [...done], updatedAt: new Date().toISOString() });102 }103 }104105 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {106 const p = PdfPayloadSchema.parse(raw.payload);107 if (!p.saleDate) return [];108 const saleDate = new Date(p.saleDate);109 const pdfName = p.url.split('/').pop()!.replace(/\.pdf$/i, '');110 // Group multi-row lots: the first row of a lot carries the prices; following rows list the other wines.111 const groups = new Map<string, Row[]>();112 for (const r of p.rows) (groups.get(r.lot) ?? groups.set(r.lot, []).get(r.lot)!).push(r);113 const out: NormalizedRecord[] = [];114 for (const [lot, rows] of groups) {115 const head = rows.find((r) => r.hammer) ?? rows[0]!;116 if (!head.hammer || head.hammer <= 0) continue;117 const f = wineFacts(head.description);118 const bundle = rows.length > 1;119 const title = bundle ? `${head.description} (+${rows.length - 1} more)` : head.description;120 const attributes = AssetAttributesSchema.parse({121 categorySlug: f.categorySlug,122 brand: f.producer,123 name: f.name,124 year: f.vintage,125 size: f.size,126 identifiers: { hdh_lot: `${pdfName}:${lot}` },127 metadata: { auction: p.auctionName, estimate: head.estimate, aggregate_usd: head.aggregate, bottles: head.qty, lines: rows.map((r) => `${r.qty ?? ''} × ${r.description}`.trim()) },128 });129 out.push(130 NormalizedSaleSchema.parse({131 kind: 'sale',132 connectorId: this.meta.id,133 sourceId: this.meta.sourceId,134 sourceUrl: `${p.url}#lot-${lot}`,135 externalId: `${pdfName}:${lot}`,136 rawTitle: title,137 imageUrls: [],138 attributes,139 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },140 condition: { condition: null, conditionRaw: null, completeness: null },141 observedAt: raw.fetchedAt,142 confidence: 0.92,143 parserVersion: PARSER_VERSION,144 saleType: 'auction',145 saleDate,146 price: head.hammer,147 currency: 'USD',148 buyerPremiumIncluded: false,149 quantity: head.qty ?? 1,150 isBundle: bundle || (head.qty ?? 1) > 1,151 location: 'Chicago, IL, United States',152 auctionHouse: 'Hart Davis Hart Wine Co.',153 lotNumber: lot,154 }),155 );156 }157 return out;158 }159}160161export default function createConnector(meta: ConnectorMeta): HdhWineConnector {162 return new HdhWineConnector(meta);163}164