import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord } from '@rareindex/shared'; import { dateWords, makeSale, money, vehicleAttributes } from '../../firecrawl/_carlib/index.js'; /** * The Market by Bonhams — server-rendered results grid. One raw record per results page; one sale per card * carrying "Sold for on ". */ const BASE = 'https://www.themarket.co.uk'; const PARSER_VERSION = '1.0.0'; export const CardSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), intro: z.string().nullable(), soldText: z.string(), bids: z.number().nullable(), location: z.string().nullable(), image: z.string().nullable() }); export type Card = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), page: z.number(), url: z.string(), cards: z.array(CardSchema) }); export function parseResultsPage(htmlText: string): Card[] { const $ = H.load(htmlText); const out: Card[] = []; $('a[data-qa="listing card"]').each((_, a) => { const href = $(a).attr('href') ?? ''; const id = href.match(/\/listings\/[^/]+\/[^/]+\/([0-9a-f-]{36})/)?.[1]; const title = H.text($(a).find('.listing-title')); const soldText = H.text($(a).find('.listing-card__heading .heading-text')); if (!id || !title || !soldText || !/^Sold for/i.test(soldText)) return; const bidsTxt = H.text($(a).find('.bids-count')); const img = $(a).find('img').attr('src') ?? null; out.push({ id, url: BASE + href, title, intro: H.text($(a).find('.listing-intro')), soldText, bids: bidsTxt ? Number(bidsTxt.replace(/\D/g, '')) || null : null, location: H.text($(a).find('.listing-location .text')), image: img ? img.replace(/\?.*$/, '') : null, }); }); return out; } /** "Sold for £56,000 on 17 Aug 2026" → price/currency/date. */ export function parseSoldText(s: string): { amount: number; currency: 'GBP' | 'EUR' | 'AUD' | 'USD'; date: Date } | null { const m = s.match(/Sold for\s+(A\$|US\$|\$|£|€)\s?([\d,]+(?:\.\d+)?)\s+on\s+(.+)$/i); if (!m) return null; const sym = m[1]!; const currency = sym === '£' ? 'GBP' : sym === '€' ? 'EUR' : sym === 'A$' ? 'AUD' : 'USD'; const amt = money(`${m[2]} ${currency}`, currency); const date = dateWords(m[3]!); if (!amt || !date) return null; return { amount: amt.amount, currency, date }; } export class TheMarketConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; async *crawl(ctx: CrawlContext): AsyncIterable { const pages = Number(this.meta.config.pagesPerRun ?? 8); const backfill = ctx.options.mode === 'backfill'; const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1; const newestSeen = !backfill && typeof ctx.options.cursor?.newestDate === 'string' ? String(ctx.options.cursor.newestDate) : ''; let maxDate = newestSeen; let count = 0; for (let page = start; page < start + pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${BASE}/auctions/results?page=${page}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'date', 'status'], parse: (r) => { const c = r.html ? parseResultsPage(r.html) : []; const p = c[0] ? parseSoldText(c[0].soldText) : null; return c.length ? { title: c[0]!.title, price: p?.amount ?? null, date: p?.date ?? null, status: 'sold' } : null; }, }); const cards = res.success && res.html ? parseResultsPage(res.html) : []; if (!cards.length) { ctx.anomaly(res.success ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const dates = cards.map((c) => parseSoldText(c.soldText)?.date.toISOString() ?? '').filter(Boolean); for (const d of dates) if (d > maxDate) maxDate = d; count++; yield { url, externalId: `results:${page}:${cards[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'results_page' as const, page, url, cards }, fetchedAt: res.fetchedAt }; const oldest = dates.sort()[0] ?? ''; if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() }); else if (newestSeen && oldest && oldest <= newestSeen) break; } if (!backfill && maxDate) await ctx.setCursor({ newestDate: maxDate, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const c of p.cards) { const sold = parseSoldText(c.soldText); if (!sold) continue; const country = c.location?.match(/,\s*([A-Z]{2})$/)?.[1] ?? null; const attributes = vehicleAttributes(c.title, { country, identifiers: { themarket_id: c.id }, metadata: { intro: c.intro, bids: c.bids, location: c.location } }); out.push( makeSale({ meta: this.meta, sourceUrl: c.url, externalId: c.id, rawTitle: c.title, attributes, price: sold.amount, currency: sold.currency, saleDate: sold.date, buyerPremiumIncluded: false, auctionHouse: 'The Market by Bonhams', imageUrls: c.image ? [c.image] : [], location: c.location, description: c.intro, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): TheMarketConnector { return new TheMarketConnector(meta); }