import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, extractYear, type NormalizedRecord } from '@rareindex/shared'; /** * Lyon & Turnbull — auction results embedded as JSON in each auction page. One raw record per * auction (compact lot list); normalise → one sale per sold lot (hammer price, GBP) and one * auction_lot for unsold lots. */ const BASE = 'https://www.lyonandturnbull.com'; const PARSER_VERSION = '1.0.0'; export const PastAuctionSchema = z.object({ saleNumber: z.string(), title: z.string(), subtitle: z.string().nullable(), date: z.string().nullable(), amsDate: z.string().nullable(), link: z.string() }); export type PastAuction = z.infer; export const LotSchema = z.object({ uuid: z.string(), lotNo: z.string(), title: z.string(), subTitle: z.string().nullable(), low: z.number().nullable(), high: z.number().nullable(), hammer: z.number().nullable(), status: z.string().nullable(), sessionDate: z.string().nullable(), image: z.string().nullable(), }); export const AuctionPayloadSchema = z.object({ kind: z.literal('auction_results'), url: z.string(), auction: PastAuctionSchema, currency: z.string(), status: z.string().nullable(), premiumTiers: z.array(z.object({ percent: z.number(), amount_over: z.number() })), lots: z.array(LotSchema), }); export type AuctionPayload = z.infer; /** Past auctions from the /auctions/past-auctions __NEXT_DATA__ (`auctionsListData`). */ export function parsePastAuctions(htmlText: string): PastAuction[] { const data = H.nextData(htmlText) as { props?: { pageProps?: { auctionsListData?: Array> } } } | null; const list = data?.props?.pageProps?.auctionsListData ?? []; return list .map((a) => ({ saleNumber: String(a.saleNumber ?? ''), title: String(a.title ?? '').trim(), subtitle: a.subtitle ? String(a.subtitle) : null, date: a.date ? String(a.date) : null, amsDate: a.amsDate ? String(a.amsDate) : null, link: String(a.arrowLink ?? ''), })) .filter((a) => a.link.startsWith('/auctions/') && a.saleNumber); } function num(v: unknown): number | null { if (v === null || v === undefined || v === '') return null; const n = Number(v); return Number.isFinite(n) && n > 0 ? n : null; } /** Extract the embedded lots JSON ("lots":[...]) plus auction status/currency/premium tiers from an auction page. */ export function parseAuctionPage(htmlText: string, url: string, auction: PastAuction): AuctionPayload | null { const data = H.nextData(htmlText) as Record | null; let lotsRaw: Array> | null = null; let auctionRaw: Record | null = null; const visit = (o: unknown, depth: number): void => { if (!o || typeof o !== 'object' || depth > 8 || (lotsRaw && auctionRaw)) return; if (Array.isArray(o)) { for (const x of o) visit(x, depth + 1); return; } const rec = o as Record; if (!lotsRaw && Array.isArray(rec.lots) && rec.lots.length && typeof (rec.lots[0] as Record).lot_no !== 'undefined') { lotsRaw = rec.lots as Array>; auctionRaw = rec; } for (const v of Object.values(rec)) visit(v, depth + 1); }; if (data) visit(data, 0); if (!lotsRaw) { // fallback: the JSON may be inlined outside __NEXT_DATA__ const i = htmlText.indexOf('"lots":['); if (i < 0) return null; const literal = H.inlineJson(htmlText.slice(i - 1), '"lots"'); if (!literal || !Array.isArray((literal as { lots?: unknown }).lots)) return null; lotsRaw = (literal as { lots: Array> }).lots; auctionRaw = literal as Record; } const a = auctionRaw as Record; const first = lotsRaw![0]!; // Premium tiers live on the auction object (sibling of `lots`) or on each lot's financeItem. const tierSource = (Array.isArray(a.premiums) ? a.premiums : Array.isArray((a.auction as { premiums?: unknown } | undefined)?.premiums) ? (a.auction as { premiums: unknown[] }).premiums : Array.isArray((first.financeItem as { premium_tiers?: unknown } | undefined)?.premium_tiers) ? (first.financeItem as { premium_tiers: unknown[] }).premium_tiers : []) as Array<{ percent?: number; amount_over?: number }>; const premiums = tierSource.map((p) => ({ percent: Number(p.percent ?? 0), amount_over: Number(p.amount_over ?? 0) })); const currencySymbol = ((first.auction as { currency?: { symbol?: string } } | undefined)?.currency?.symbol ?? (a.currency as { symbol?: string } | undefined)?.symbol ?? '£'); const currency = currencySymbol === '£' ? 'GBP' : currencySymbol === '€' ? 'EUR' : currencySymbol === '$' ? 'USD' : 'GBP'; const status = typeof a.status === 'string' ? a.status : ((first.auction as { status?: string } | undefined)?.status ?? null); const lots = lotsRaw!.map((l) => { const title = (l.title as { en?: string } | undefined)?.en ?? (typeof l.title === 'string' ? l.title : ''); const sub = ((l.dynamic_fields as { en?: { sub_title?: string } } | undefined)?.en?.sub_title ?? null) as string | null; const img = (l.images as Array<{ data?: { original?: { url?: string }; sm?: { url?: string } } }> | undefined)?.[0]?.data; return { uuid: String(l.uuid ?? ''), lotNo: String(l.lot_no ?? ''), title: String(title).trim(), subTitle: sub ? String(sub).trim() : null, low: num(l.low), high: num(l.high), hammer: num(l.hammer_price) ?? num((l.financeItem as { hammer_price?: unknown } | undefined)?.hammer_price), status: typeof l.status === 'string' ? l.status : null, sessionDate: typeof l.session_date === 'string' ? l.session_date : null, image: img?.original?.url ?? img?.sm?.url ?? null, }; }); return { kind: 'auction_results', url, auction, currency, status, premiumTiers: premiums, lots: lots.filter((l) => l.title && l.lotNo) }; } const TITLE_CATEGORY: Array<[RegExp, string]> = [ [/jewel/i, 'jewelry'], [/silver|objets de vertu/i, 'silver'], [/whisky|spirits|wine/i, 'whisky'], [/watch|clock|horolog/i, 'watches'], [/book|manuscript|map/i, 'books'], [/photograph/i, 'photography'], [/contemporary|modern made|post-war|prints|editions|modern british/i, 'contemporary_art'], [/design|art deco|art nouveau|decorative arts|furniture|form through time/i, 'design_furniture'], [/asian|chinese|japanese|islamic|african|tribal|ceramic|porcelain|glass/i, 'antiques'], [/scottish|paintings|pictures|old master|fine art|art/i, 'art'], ]; export function categoryForAuction(title: string): string { return TITLE_CATEGORY.find(([re]) => re.test(title))?.[1] ?? 'antiques'; } /** Lot-level refinements for cross-category sales (natural history, watches inside mixed auctions…). */ const LOT_CATEGORY: Array<[RegExp, string]> = [ [/meteorite/i, 'meteorites'], [/fossil|ammonite|trilobite|dinosaur|mammoth|megalodon/i, 'fossils'], [/mineral|crystal|quartz|amethyst|geode|specimen/i, 'minerals'], [/wristwatch|pocket watch|chronograph/i, 'other_watches'], [/rolex/i, 'rolex'], [/patek philippe/i, 'patek_philippe'], [/omega/i, 'omega'], ]; export function categoryForLot(auctionCategory: string, title: string): string { return LOT_CATEGORY.find(([re]) => re.test(title))?.[1] ?? auctionCategory; } /** Buyer's premium for a hammer price from tiered percentages (each tier applies to the slice above `amount_over`). */ export function premiumFor(hammer: number, tiers: Array<{ percent: number; amount_over: number }>): number | null { if (!tiers.length) return null; const sorted = [...tiers].sort((a, b) => a.amount_over - b.amount_over); let total = 0; for (let i = 0; i < sorted.length; i++) { const from = sorted[i]!.amount_over; const to = sorted[i + 1]?.amount_over ?? Infinity; if (hammer <= from) break; total += (Math.min(hammer, to) - from) * (sorted[i]!.percent / 100); } return Math.round(total * 100) / 100; } export class LyonTurnbullConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const pastUrl = String(this.meta.config.pastAuctionsUrl ?? `${BASE}/auctions/past-auctions`); const perRun = Number(this.meta.config.auctionsPerRun ?? 3); const cursor = (ctx.options.cursor ?? {}) as { done?: string[] }; const done = new Set(cursor.done ?? []); const list = await ctx.fetch(pastUrl, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); if (!list.success || !list.html) { ctx.anomaly('past_list_failed', list.error ?? String(list.httpStatus)); return; } const auctions = parsePastAuctions(list.html).sort((a, b) => (b.amsDate ?? '').localeCompare(a.amsDate ?? '')); let fetched = 0; let count = 0; for (const auction of auctions) { if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun) break; if (done.has(auction.saleNumber) && ctx.options.mode !== 'backfill') continue; const url = BASE + auction.link; await this.throttle(); fetched++; const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', timeoutMs: 60_000, expect: ['title', 'price', 'currency', 'status'], parse: (r) => { const p = r.html ? parseAuctionPage(r.html, url, auction) : null; const sold = p?.lots.find((l) => l.hammer); return p ? { title: p.lots[0]?.title, price: sold?.hammer ?? null, currency: sold ? p.currency : null, status: p.status } : null; }, }); const payload = res.success && res.html ? parseAuctionPage(res.html, url, auction) : null; if (!payload) { ctx.anomaly('auction_parse_failed', `${auction.saleNumber}: ${res.error ?? res.httpStatus}`); continue; } if (payload.status && payload.status !== 'completed') { // still live/published — results not final yet continue; } count++; done.add(auction.saleNumber); await ctx.setCursor({ done: [...done].slice(-400) }); yield { url, externalId: auction.saleNumber, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } } async normalize(raw: RawRecordLike): Promise { const p = AuctionPayloadSchema.parse(raw.payload); const categorySlug = categoryForAuction(`${p.auction.title} ${p.auction.subtitle ?? ''}`); const out: NormalizedRecord[] = []; for (const lot of p.lots) { const saleDateIso = lot.sessionDate ?? p.auction.amsDate; const saleDate = saleDateIso ? new Date(saleDateIso) : null; const lotUrl = `${BASE}${p.auction.link}/lot/${lot.lotNo}`; const yearHint = lot.subTitle ? extractYear(lot.subTitle) : null; const attributes = AssetAttributesSchema.parse({ categorySlug: categoryForLot(categorySlug, `${lot.title} ${lot.subTitle ?? ''}`), name: lot.title, year: yearHint, identifiers: { lt_lot_uuid: lot.uuid, lt_lot: `${p.auction.saleNumber}/${lot.lotNo}` }, metadata: { sale_number: p.auction.saleNumber, auction_title: p.auction.title, sub_title: lot.subTitle, estimate_low: lot.low, estimate_high: lot.high, premium_tiers: p.premiumTiers, premium_estimate: lot.hammer ? premiumFor(lot.hammer, p.premiumTiers) : null }, }); const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: lotUrl, externalId: `${p.auction.saleNumber}:${lot.lotNo}`, rawTitle: lot.subTitle ? `${lot.title} — ${lot.subTitle}` : lot.title, description: null, imageUrls: lot.image ? [lot.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: PARSER_VERSION, }; if (lot.hammer && saleDate && lot.status !== 'unsold') { out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate, price: lot.hammer, currency: p.currency, buyerPremiumIncluded: false, quantity: 1, isBundle: false, location: 'United Kingdom', auctionHouse: 'Lyon & Turnbull', lotNumber: lot.lotNo })); } else { out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: 'Lyon & Turnbull', auctionName: p.auction.title, lotNumber: lot.lotNo, startsAt: saleDate, endsAt: saleDate, estimateLow: lot.low, estimateHigh: lot.high, currentBid: null, currency: p.currency, status: 'ended', location: 'United Kingdom' })); } } return out; } } export default function createConnector(meta: ConnectorMeta) { return new LyonTurnbullConnector(meta); }