TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { dateWords, makeSale, money, vehicleAttributes } from '../../firecrawl/_carlib/index.js';56/**7 * The Market by Bonhams — server-rendered results grid. One raw record per results page; one sale per card8 * carrying "Sold for <price> on <date>".9 */10const BASE = 'https://www.themarket.co.uk';11const PARSER_VERSION = '1.0.0';1213export const CardSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), intro: z.string().nullable(), soldText: z.string(), bids: z.number().nullable(), location: z.string().nullable(), image: z.string().nullable() });14export type Card = z.infer<typeof CardSchema>;15export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), page: z.number(), url: z.string(), cards: z.array(CardSchema) });1617export function parseResultsPage(htmlText: string): Card[] {18 const $ = H.load(htmlText);19 const out: Card[] = [];20 $('a[data-qa="listing card"]').each((_, a) => {21 const href = $(a).attr('href') ?? '';22 const id = href.match(/\/listings\/[^/]+\/[^/]+\/([0-9a-f-]{36})/)?.[1];23 const title = H.text($(a).find('.listing-title'));24 const soldText = H.text($(a).find('.listing-card__heading .heading-text'));25 if (!id || !title || !soldText || !/^Sold for/i.test(soldText)) return;26 const bidsTxt = H.text($(a).find('.bids-count'));27 const img = $(a).find('img').attr('src') ?? null;28 out.push({29 id,30 url: BASE + href,31 title,32 intro: H.text($(a).find('.listing-intro')),33 soldText,34 bids: bidsTxt ? Number(bidsTxt.replace(/\D/g, '')) || null : null,35 location: H.text($(a).find('.listing-location .text')),36 image: img ? img.replace(/\?.*$/, '') : null,37 });38 });39 return out;40}4142/** "Sold for £56,000 on 17 Aug 2026" → price/currency/date. */43export function parseSoldText(s: string): { amount: number; currency: 'GBP' | 'EUR' | 'AUD' | 'USD'; date: Date } | null {44 const m = s.match(/Sold for\s+(A\$|US\$|\$|£|€)\s?([\d,]+(?:\.\d+)?)\s+on\s+(.+)$/i);45 if (!m) return null;46 const sym = m[1]!;47 const currency = sym === '£' ? 'GBP' : sym === '€' ? 'EUR' : sym === 'A$' ? 'AUD' : 'USD';48 const amt = money(`${m[2]} ${currency}`, currency);49 const date = dateWords(m[3]!);50 if (!amt || !date) return null;51 return { amount: amt.amount, currency, date };52}5354export class TheMarketConnector extends BaseConnector {55 readonly version = '1.0.0';56 readonly parserVersion = PARSER_VERSION;57 protected override minIntervalMs = 1500;5859 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {60 const pages = Number(this.meta.config.pagesPerRun ?? 8);61 const backfill = ctx.options.mode === 'backfill';62 const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1;63 const newestSeen = !backfill && typeof ctx.options.cursor?.newestDate === 'string' ? String(ctx.options.cursor.newestDate) : '';64 let maxDate = newestSeen;65 let count = 0;66 for (let page = start; page < start + pages; page++) {67 if (ctx.signal?.aborted || this.reached(ctx, count)) break;68 const url = `${BASE}/auctions/results?page=${page}`;69 await this.throttle();70 const res = await ctx.fetch(url, {71 engines: ['api'],72 responseType: 'text',73 expect: ['title', 'price', 'date', 'status'],74 parse: (r) => {75 const c = r.html ? parseResultsPage(r.html) : [];76 const p = c[0] ? parseSoldText(c[0].soldText) : null;77 return c.length ? { title: c[0]!.title, price: p?.amount ?? null, date: p?.date ?? null, status: 'sold' } : null;78 },79 });80 const cards = res.success && res.html ? parseResultsPage(res.html) : [];81 if (!cards.length) {82 ctx.anomaly(res.success ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);83 break;84 }85 const dates = cards.map((c) => parseSoldText(c.soldText)?.date.toISOString() ?? '').filter(Boolean);86 for (const d of dates) if (d > maxDate) maxDate = d;87 count++;88 yield { url, externalId: `results:${page}:${cards[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'results_page' as const, page, url, cards }, fetchedAt: res.fetchedAt };89 const oldest = dates.sort()[0] ?? '';90 if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() });91 else if (newestSeen && oldest && oldest <= newestSeen) break;92 }93 if (!backfill && maxDate) await ctx.setCursor({ newestDate: maxDate, updatedAt: new Date().toISOString() });94 }9596 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {97 const p = PagePayloadSchema.parse(raw.payload);98 const out: NormalizedRecord[] = [];99 for (const c of p.cards) {100 const sold = parseSoldText(c.soldText);101 if (!sold) continue;102 const country = c.location?.match(/,\s*([A-Z]{2})$/)?.[1] ?? null;103 const attributes = vehicleAttributes(c.title, { country, identifiers: { themarket_id: c.id }, metadata: { intro: c.intro, bids: c.bids, location: c.location } });104 out.push(105 makeSale({106 meta: this.meta,107 sourceUrl: c.url,108 externalId: c.id,109 rawTitle: c.title,110 attributes,111 price: sold.amount,112 currency: sold.currency,113 saleDate: sold.date,114 buyerPremiumIncluded: false,115 auctionHouse: 'The Market by Bonhams',116 imageUrls: c.image ? [c.image] : [],117 location: c.location,118 description: c.intro,119 observedAt: raw.fetchedAt,120 parserVersion: PARSER_VERSION,121 }),122 );123 }124 return out;125 }126}127128export default function createConnector(meta: ConnectorMeta): TheMarketConnector {129 return new TheMarketConnector(meta);130}131