TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { lotAttributes, makeSale, money, vehicleAttributes } from '../../firecrawl/_carlib/index.js';56/**7 * Historics Auctioneers — results list → per-sale "Past lots" grid (96 lots per page) with "Sold £X".8 * One raw record per lots page; one sale per lot with a price.9 */10const BASE = 'https://www.historics.co.uk';11const PARSER_VERSION = '1.0.0';1213export const AuctionSchema = z.object({ au: z.string(), slug: z.string(), title: z.string(), saleNumber: z.string().nullable(), endedOn: z.string().nullable(), lotCount: z.number().nullable() });14export type Auction = z.infer<typeof AuctionSchema>;15export const LotSchema = z.object({ lotId: z.string(), url: z.string(), lotNo: z.string().nullable(), title: z.string(), subtitle: z.string().nullable(), soldText: z.string().nullable(), priceGbp: z.number().nullable(), image: z.string().nullable() });16export type Lot = z.infer<typeof LotSchema>;17export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), url: z.string(), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) });1819const MONTHS: Record<string, number> = { jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11 };20/** "18th Jul, 2026 9:30" | "6th Aug, 2026 19:30" → UTC midnight. */21export function parseUkDate(s: string | null | undefined): Date | null {22 const m = s?.match(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3})[a-z]*,?\s+(\d{4})/);23 if (!m) return null;24 const mo = MONTHS[m[2]!.toLowerCase()];25 return mo === undefined ? null : new Date(Date.UTC(Number(m[3]), mo, Number(m[1])));26}2728/** Parse /auction-results: calendar items with title, "Date:"/"Ends:" and "Sale number:". */29export function parseResultsList(htmlText: string): Auction[] {30 const $ = H.load(htmlText);31 const out: Auction[] = [];32 const seen = new Set<string>();33 $('.auction-calendar-item').each((_, el) => {34 const href = $(el).find('a[href*="/auction/details/"]').first().attr('href') ?? '';35 const m = href.match(/\/auction\/details\/([^/?]+)\?au=(\d+)/);36 if (!m || seen.has(m[2]!)) return;37 const text = $(el).text().replace(/\s+/g, ' ').trim();38 const title = H.text($(el).find('.auction-calendar-text h2, .auction-calendar-text h3, .auction-calendar-text h4').first()) ?? text.split(/\s(?:Date|Ends):/)[0]!.trim();39 const dateTxt = text.match(/(?:Date|Ends):\s*([^S]+?)\s+Sale number/i)?.[1] ?? text.match(/(?:Date|Ends):\s*(\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]{3,9},?\s+\d{4})/i)?.[1] ?? null;40 const saleNumber = text.match(/Sale number:\s*([A-Z]{1,2}O?\d{2,4})/i)?.[1] ?? null;41 const lots = text.match(/Lots:\s*(\d+)/i)?.[1];42 seen.add(m[2]!);43 out.push({ au: m[2]!, slug: m[1]!, title, saleNumber, endedOn: parseUkDate(dateTxt)?.toISOString() ?? null, lotCount: lots ? Number(lots) : null });44 });45 return out;46}4748/** Parse a lots page: cards with lot number, title, "Sold £X". */49export function parseLotsPage(htmlText: string): Lot[] {50 const $ = H.load(htmlText);51 const out: Lot[] = [];52 $('.auction-lot').each((_, el) => {53 const a = $(el).find('.auction-lot-title a').first();54 const href = a.attr('href') ?? '';55 const lotId = href.match(/[?&]lot=(\d+)/)?.[1];56 if (!lotId) return;57 const titleEl = a.find('.lot-title').clone();58 const subtitle = H.text(titleEl.find('.sub-title'));59 titleEl.find('.sub-title').remove();60 const full = H.text(titleEl) ?? '';61 const lotNo = full.match(/^Lot\s+([A-Z]?\d+[A-Z]?)\s*-\s*/i)?.[1] ?? null;62 const title = full.replace(/^Lot\s+[A-Z]?\d+[A-Z]?\s*-\s*/i, '').trim();63 if (!title) return;64 const soldText = H.text($(el).find('strong').filter((_, s) => /^Sold/i.test($(s).text().trim())).first());65 const priceTxt = soldText?.match(/Sold\s*£\s*([\d,]+(?:\.\d+)?)/i)?.[1] ?? null;66 const img = $(el).find('.auction-lot-image img').attr('src') ?? null;67 out.push({ lotId, url: `${BASE}${href.split('&so=')[0]!.replace(/&/g, '&')}`, lotNo, title, subtitle, soldText, priceGbp: priceTxt ? Number(priceTxt.replace(/,/g, '')) : null, image: img ? img.replace(/\?v=.*$/, '') : null });68 });69 return out;70}7172export function categoryFor(auction: Auction, title: string): { categorySlug: string; vehicle: boolean } {73 const sale = (auction.saleNumber ?? '').toUpperCase();74 if (/registration|number plate|cherished/i.test(title) || /^[A-Z]{1,2}\d*\s*[A-Z]{0,3}$/.test(title.replace(/\s+/g, ' ').trim()) && sale.startsWith('W')) return { categorySlug: 'license_plates', vehicle: false };75 if (sale.startsWith('W') && !/^(19|20)\d{2}\b/.test(title)) return { categorySlug: 'automotive_memorabilia', vehicle: false };76 return { categorySlug: 'automobiles', vehicle: true };77}7879export class HistoricsConnector extends BaseConnector {80 readonly version = '1.0.0';81 readonly parserVersion = PARSER_VERSION;82 protected override minIntervalMs = 10_000; // robots.txt crawl-delay: 108384 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {85 const perRun = Number(this.meta.config.auctionsPerRun ?? 1);86 const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 3);87 const progress = { ...((ctx.options.cursor?.progress as Record<string, number> | undefined) ?? {}) };88 const complete = new Set<string>((ctx.options.cursor?.complete as string[] | undefined) ?? []);89 await this.throttle();90 const list = await ctx.fetch(`${BASE}/auction-results`, { engines: ['api'], responseType: 'text', expect: ['title', 'date'], parse: (r) => (r.html ? { title: parseResultsList(r.html)[0]?.title, date: parseResultsList(r.html)[0]?.endedOn } : null) });91 if (!list.success || !list.html) {92 ctx.anomaly('page_fetch_failed', `results list: ${list.error ?? list.httpStatus}`);93 return;94 }95 const auctions = parseResultsList(list.html).filter((a) => a.endedOn && new Date(a.endedOn).getTime() < Date.now());96 const todo = auctions.filter((a) => !complete.has(a.au)).slice(0, perRun);97 let pagesFetched = 0;98 let count = 0;99 for (const auction of todo) {100 let page = (progress[auction.au] ?? 0) + 1;101 while (true) {102 if (ctx.signal?.aborted || this.reached(ctx, count) || pagesFetched >= pagesPerRun) return void (await ctx.setCursor({ progress, complete: [...complete] }));103 const url = `${BASE}/auction/details/${auction.slug}?au=${auction.au}&pp=96&pn=${page}`;104 await this.throttle();105 const res = await ctx.fetch(url, {106 engines: ['api'],107 responseType: 'text',108 expect: ['title', 'price', 'currency', 'status'],109 parse: (r) => {110 const lots = r.html ? parseLotsPage(r.html) : [];111 const sold = lots.find((l) => l.priceGbp);112 return lots.length ? { title: lots[0]!.title, price: sold?.priceGbp ?? null, currency: sold ? 'GBP' : null, status: sold ? 'sold' : null } : null;113 },114 });115 pagesFetched++;116 const lots = res.success && res.html ? parseLotsPage(res.html) : [];117 if (!lots.length) {118 if (!res.success) ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);119 complete.add(auction.au);120 break;121 }122 count++;123 yield { url, externalId: `auction:${auction.au}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, auction, page, lots }, fetchedAt: res.fetchedAt };124 progress[auction.au] = page;125 if (lots.length < 96) {126 complete.add(auction.au);127 break;128 }129 page++;130 }131 }132 await ctx.setCursor({ progress, complete: [...complete] });133 }134135 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {136 const p = PagePayloadSchema.parse(raw.payload);137 if (!p.auction.endedOn) return [];138 const saleDate = new Date(p.auction.endedOn);139 const out: NormalizedRecord[] = [];140 for (const lot of p.lots) {141 if (!lot.priceGbp || lot.priceGbp <= 0) continue;142 const m = money(`£${lot.priceGbp}`, 'GBP');143 if (!m) continue;144 const cat = categoryFor(p.auction, lot.title);145 const meta = { auction: p.auction.title, sale_number: p.auction.saleNumber, subtitle: lot.subtitle, sold_text: lot.soldText };146 const attributes = cat.vehicle ? vehicleAttributes(lot.title, { country: 'GB', identifiers: { historics_lot: lot.lotId }, metadata: meta }) : lotAttributes({ categorySlug: cat.categorySlug, name: lot.title, country: 'GB', identifiers: { historics_lot: lot.lotId }, metadata: meta });147 out.push(148 makeSale({149 meta: this.meta,150 sourceUrl: lot.url,151 externalId: lot.lotId,152 rawTitle: lot.title,153 attributes,154 price: m.amount,155 currency: 'GBP',156 saleDate,157 buyerPremiumIncluded: false,158 auctionHouse: 'Historics Auctioneers',159 lotNumber: lot.lotNo,160 imageUrls: lot.image ? [lot.image] : [],161 description: lot.subtitle,162 location: 'United Kingdom',163 observedAt: raw.fetchedAt,164 parserVersion: PARSER_VERSION,165 }),166 );167 }168 return out;169 }170}171172export default function createConnector(meta: ConnectorMeta): HistoricsConnector {173 return new HistoricsConnector(meta);174}175