import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord } from '@rareindex/shared'; import { lotAttributes, makeSale, money, vehicleAttributes } from '../../firecrawl/_carlib/index.js'; /** * Historics Auctioneers — results list → per-sale "Past lots" grid (96 lots per page) with "Sold £X". * One raw record per lots page; one sale per lot with a price. */ const BASE = 'https://www.historics.co.uk'; const PARSER_VERSION = '1.0.0'; export const AuctionSchema = z.object({ au: z.string(), slug: z.string(), title: z.string(), saleNumber: z.string().nullable(), endedOn: z.string().nullable(), lotCount: z.number().nullable() }); export type Auction = z.infer; export const LotSchema = z.object({ lotId: z.string(), url: z.string(), lotNo: z.string().nullable(), title: z.string(), subtitle: z.string().nullable(), soldText: z.string().nullable(), priceGbp: z.number().nullable(), image: z.string().nullable() }); export type Lot = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), url: z.string(), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) }); const MONTHS: Record = { jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11 }; /** "18th Jul, 2026 9:30" | "6th Aug, 2026 19:30" → UTC midnight. */ export function parseUkDate(s: string | null | undefined): Date | null { const m = s?.match(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3})[a-z]*,?\s+(\d{4})/); if (!m) return null; const mo = MONTHS[m[2]!.toLowerCase()]; return mo === undefined ? null : new Date(Date.UTC(Number(m[3]), mo, Number(m[1]))); } /** Parse /auction-results: calendar items with title, "Date:"/"Ends:" and "Sale number:". */ export function parseResultsList(htmlText: string): Auction[] { const $ = H.load(htmlText); const out: Auction[] = []; const seen = new Set(); $('.auction-calendar-item').each((_, el) => { const href = $(el).find('a[href*="/auction/details/"]').first().attr('href') ?? ''; const m = href.match(/\/auction\/details\/([^/?]+)\?au=(\d+)/); if (!m || seen.has(m[2]!)) return; const text = $(el).text().replace(/\s+/g, ' ').trim(); const title = H.text($(el).find('.auction-calendar-text h2, .auction-calendar-text h3, .auction-calendar-text h4').first()) ?? text.split(/\s(?:Date|Ends):/)[0]!.trim(); const dateTxt = text.match(/(?:Date|Ends):\s*([^S]+?)\s+Sale number/i)?.[1] ?? text.match(/(?:Date|Ends):\s*(\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]{3,9},?\s+\d{4})/i)?.[1] ?? null; const saleNumber = text.match(/Sale number:\s*([A-Z]{1,2}O?\d{2,4})/i)?.[1] ?? null; const lots = text.match(/Lots:\s*(\d+)/i)?.[1]; seen.add(m[2]!); out.push({ au: m[2]!, slug: m[1]!, title, saleNumber, endedOn: parseUkDate(dateTxt)?.toISOString() ?? null, lotCount: lots ? Number(lots) : null }); }); return out; } /** Parse a lots page: cards with lot number, title, "Sold £X". */ export function parseLotsPage(htmlText: string): Lot[] { const $ = H.load(htmlText); const out: Lot[] = []; $('.auction-lot').each((_, el) => { const a = $(el).find('.auction-lot-title a').first(); const href = a.attr('href') ?? ''; const lotId = href.match(/[?&]lot=(\d+)/)?.[1]; if (!lotId) return; const titleEl = a.find('.lot-title').clone(); const subtitle = H.text(titleEl.find('.sub-title')); titleEl.find('.sub-title').remove(); const full = H.text(titleEl) ?? ''; const lotNo = full.match(/^Lot\s+([A-Z]?\d+[A-Z]?)\s*-\s*/i)?.[1] ?? null; const title = full.replace(/^Lot\s+[A-Z]?\d+[A-Z]?\s*-\s*/i, '').trim(); if (!title) return; const soldText = H.text($(el).find('strong').filter((_, s) => /^Sold/i.test($(s).text().trim())).first()); const priceTxt = soldText?.match(/Sold\s*£\s*([\d,]+(?:\.\d+)?)/i)?.[1] ?? null; const img = $(el).find('.auction-lot-image img').attr('src') ?? null; out.push({ lotId, url: `${BASE}${href.split('&so=')[0]!.replace(/&/g, '&')}`, lotNo, title, subtitle, soldText, priceGbp: priceTxt ? Number(priceTxt.replace(/,/g, '')) : null, image: img ? img.replace(/\?v=.*$/, '') : null }); }); return out; } export function categoryFor(auction: Auction, title: string): { categorySlug: string; vehicle: boolean } { const sale = (auction.saleNumber ?? '').toUpperCase(); if (/registration|number plate|cherished/i.test(title) || /^[A-Z]{1,2}\d*\s*[A-Z]{0,3}$/.test(title.replace(/\s+/g, ' ').trim()) && sale.startsWith('W')) return { categorySlug: 'license_plates', vehicle: false }; if (sale.startsWith('W') && !/^(19|20)\d{2}\b/.test(title)) return { categorySlug: 'automotive_memorabilia', vehicle: false }; return { categorySlug: 'automobiles', vehicle: true }; } export class HistoricsConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 10_000; // robots.txt crawl-delay: 10 async *crawl(ctx: CrawlContext): AsyncIterable { const perRun = Number(this.meta.config.auctionsPerRun ?? 1); const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 3); const progress = { ...((ctx.options.cursor?.progress as Record | undefined) ?? {}) }; const complete = new Set((ctx.options.cursor?.complete as string[] | undefined) ?? []); await this.throttle(); const list = await ctx.fetch(`${BASE}/auction-results`, { engines: ['api'], responseType: 'text', expect: ['title', 'date'], parse: (r) => (r.html ? { title: parseResultsList(r.html)[0]?.title, date: parseResultsList(r.html)[0]?.endedOn } : null) }); if (!list.success || !list.html) { ctx.anomaly('page_fetch_failed', `results list: ${list.error ?? list.httpStatus}`); return; } const auctions = parseResultsList(list.html).filter((a) => a.endedOn && new Date(a.endedOn).getTime() < Date.now()); const todo = auctions.filter((a) => !complete.has(a.au)).slice(0, perRun); let pagesFetched = 0; let count = 0; for (const auction of todo) { let page = (progress[auction.au] ?? 0) + 1; while (true) { if (ctx.signal?.aborted || this.reached(ctx, count) || pagesFetched >= pagesPerRun) return void (await ctx.setCursor({ progress, complete: [...complete] })); const url = `${BASE}/auction/details/${auction.slug}?au=${auction.au}&pp=96&pn=${page}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'currency', 'status'], parse: (r) => { const lots = r.html ? parseLotsPage(r.html) : []; const sold = lots.find((l) => l.priceGbp); return lots.length ? { title: lots[0]!.title, price: sold?.priceGbp ?? null, currency: sold ? 'GBP' : null, status: sold ? 'sold' : null } : null; }, }); pagesFetched++; const lots = res.success && res.html ? parseLotsPage(res.html) : []; if (!lots.length) { if (!res.success) ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); complete.add(auction.au); break; } count++; yield { url, externalId: `auction:${auction.au}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, auction, page, lots }, fetchedAt: res.fetchedAt }; progress[auction.au] = page; if (lots.length < 96) { complete.add(auction.au); break; } page++; } } await ctx.setCursor({ progress, complete: [...complete] }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); if (!p.auction.endedOn) return []; const saleDate = new Date(p.auction.endedOn); const out: NormalizedRecord[] = []; for (const lot of p.lots) { if (!lot.priceGbp || lot.priceGbp <= 0) continue; const m = money(`£${lot.priceGbp}`, 'GBP'); if (!m) continue; const cat = categoryFor(p.auction, lot.title); const meta = { auction: p.auction.title, sale_number: p.auction.saleNumber, subtitle: lot.subtitle, sold_text: lot.soldText }; const attributes = cat.vehicle ? vehicleAttributes(lot.title, { country: 'GB', identifiers: { historics_lot: lot.lotId }, metadata: meta }) : lotAttributes({ categorySlug: cat.categorySlug, name: lot.title, country: 'GB', identifiers: { historics_lot: lot.lotId }, metadata: meta }); out.push( makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: lot.lotId, rawTitle: lot.title, attributes, price: m.amount, currency: 'GBP', saleDate, buyerPremiumIncluded: false, auctionHouse: 'Historics Auctioneers', lotNumber: lot.lotNo, imageUrls: lot.image ? [lot.image] : [], description: lot.subtitle, location: 'United Kingdom', observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): HistoricsConnector { return new HistoricsConnector(meta); }