TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';45/**6 * Lyon & Turnbull — auction results embedded as JSON in each auction page. One raw record per7 * auction (compact lot list); normalise → one sale per sold lot (hammer price, GBP) and one8 * auction_lot for unsold lots.9 */1011const BASE = 'https://www.lyonandturnbull.com';12const PARSER_VERSION = '1.0.0';1314export const PastAuctionSchema = z.object({ saleNumber: z.string(), title: z.string(), subtitle: z.string().nullable(), date: z.string().nullable(), amsDate: z.string().nullable(), link: z.string() });15export type PastAuction = z.infer<typeof PastAuctionSchema>;1617export const LotSchema = z.object({18 uuid: z.string(),19 lotNo: z.string(),20 title: z.string(),21 subTitle: z.string().nullable(),22 low: z.number().nullable(),23 high: z.number().nullable(),24 hammer: z.number().nullable(),25 status: z.string().nullable(),26 sessionDate: z.string().nullable(),27 image: z.string().nullable(),28});29export const AuctionPayloadSchema = z.object({30 kind: z.literal('auction_results'),31 url: z.string(),32 auction: PastAuctionSchema,33 currency: z.string(),34 status: z.string().nullable(),35 premiumTiers: z.array(z.object({ percent: z.number(), amount_over: z.number() })),36 lots: z.array(LotSchema),37});38export type AuctionPayload = z.infer<typeof AuctionPayloadSchema>;3940/** Past auctions from the /auctions/past-auctions __NEXT_DATA__ (`auctionsListData`). */41export function parsePastAuctions(htmlText: string): PastAuction[] {42 const data = H.nextData(htmlText) as { props?: { pageProps?: { auctionsListData?: Array<Record<string, unknown>> } } } | null;43 const list = data?.props?.pageProps?.auctionsListData ?? [];44 return list45 .map((a) => ({46 saleNumber: String(a.saleNumber ?? ''),47 title: String(a.title ?? '').trim(),48 subtitle: a.subtitle ? String(a.subtitle) : null,49 date: a.date ? String(a.date) : null,50 amsDate: a.amsDate ? String(a.amsDate) : null,51 link: String(a.arrowLink ?? ''),52 }))53 .filter((a) => a.link.startsWith('/auctions/') && a.saleNumber);54}5556function num(v: unknown): number | null {57 if (v === null || v === undefined || v === '') return null;58 const n = Number(v);59 return Number.isFinite(n) && n > 0 ? n : null;60}6162/** Extract the embedded lots JSON ("lots":[...]) plus auction status/currency/premium tiers from an auction page. */63export function parseAuctionPage(htmlText: string, url: string, auction: PastAuction): AuctionPayload | null {64 const data = H.nextData(htmlText) as Record<string, unknown> | null;65 let lotsRaw: Array<Record<string, unknown>> | null = null;66 let auctionRaw: Record<string, unknown> | null = null;67 const visit = (o: unknown, depth: number): void => {68 if (!o || typeof o !== 'object' || depth > 8 || (lotsRaw && auctionRaw)) return;69 if (Array.isArray(o)) {70 for (const x of o) visit(x, depth + 1);71 return;72 }73 const rec = o as Record<string, unknown>;74 if (!lotsRaw && Array.isArray(rec.lots) && rec.lots.length && typeof (rec.lots[0] as Record<string, unknown>).lot_no !== 'undefined') {75 lotsRaw = rec.lots as Array<Record<string, unknown>>;76 auctionRaw = rec;77 }78 for (const v of Object.values(rec)) visit(v, depth + 1);79 };80 if (data) visit(data, 0);81 if (!lotsRaw) {82 // fallback: the JSON may be inlined outside __NEXT_DATA__83 const i = htmlText.indexOf('"lots":[');84 if (i < 0) return null;85 const literal = H.inlineJson(htmlText.slice(i - 1), '"lots"');86 if (!literal || !Array.isArray((literal as { lots?: unknown }).lots)) return null;87 lotsRaw = (literal as { lots: Array<Record<string, unknown>> }).lots;88 auctionRaw = literal as Record<string, unknown>;89 }90 const a = auctionRaw as Record<string, unknown>;91 const first = lotsRaw![0]!;92 // Premium tiers live on the auction object (sibling of `lots`) or on each lot's financeItem.93 const tierSource = (Array.isArray(a.premiums) ? a.premiums : Array.isArray((a.auction as { premiums?: unknown } | undefined)?.premiums) ? (a.auction as { premiums: unknown[] }).premiums : Array.isArray((first.financeItem as { premium_tiers?: unknown } | undefined)?.premium_tiers) ? (first.financeItem as { premium_tiers: unknown[] }).premium_tiers : []) as Array<{ percent?: number; amount_over?: number }>;94 const premiums = tierSource.map((p) => ({ percent: Number(p.percent ?? 0), amount_over: Number(p.amount_over ?? 0) }));95 const currencySymbol = ((first.auction as { currency?: { symbol?: string } } | undefined)?.currency?.symbol ?? (a.currency as { symbol?: string } | undefined)?.symbol ?? '£');96 const currency = currencySymbol === '£' ? 'GBP' : currencySymbol === '€' ? 'EUR' : currencySymbol === '$' ? 'USD' : 'GBP';97 const status = typeof a.status === 'string' ? a.status : ((first.auction as { status?: string } | undefined)?.status ?? null);98 const lots = lotsRaw!.map((l) => {99 const title = (l.title as { en?: string } | undefined)?.en ?? (typeof l.title === 'string' ? l.title : '');100 const sub = ((l.dynamic_fields as { en?: { sub_title?: string } } | undefined)?.en?.sub_title ?? null) as string | null;101 const img = (l.images as Array<{ data?: { original?: { url?: string }; sm?: { url?: string } } }> | undefined)?.[0]?.data;102 return {103 uuid: String(l.uuid ?? ''),104 lotNo: String(l.lot_no ?? ''),105 title: String(title).trim(),106 subTitle: sub ? String(sub).trim() : null,107 low: num(l.low),108 high: num(l.high),109 hammer: num(l.hammer_price) ?? num((l.financeItem as { hammer_price?: unknown } | undefined)?.hammer_price),110 status: typeof l.status === 'string' ? l.status : null,111 sessionDate: typeof l.session_date === 'string' ? l.session_date : null,112 image: img?.original?.url ?? img?.sm?.url ?? null,113 };114 });115 return { kind: 'auction_results', url, auction, currency, status, premiumTiers: premiums, lots: lots.filter((l) => l.title && l.lotNo) };116}117118const TITLE_CATEGORY: Array<[RegExp, string]> = [119 [/jewel/i, 'jewelry'],120 [/silver|objets de vertu/i, 'silver'],121 [/whisky|spirits|wine/i, 'whisky'],122 [/watch|clock|horolog/i, 'watches'],123 [/book|manuscript|map/i, 'books'],124 [/photograph/i, 'photography'],125 [/contemporary|modern made|post-war|prints|editions|modern british/i, 'contemporary_art'],126 [/design|art deco|art nouveau|decorative arts|furniture|form through time/i, 'design_furniture'],127 [/asian|chinese|japanese|islamic|african|tribal|ceramic|porcelain|glass/i, 'antiques'],128 [/scottish|paintings|pictures|old master|fine art|art/i, 'art'],129];130131export function categoryForAuction(title: string): string {132 return TITLE_CATEGORY.find(([re]) => re.test(title))?.[1] ?? 'antiques';133}134135/** Lot-level refinements for cross-category sales (natural history, watches inside mixed auctions…). */136const LOT_CATEGORY: Array<[RegExp, string]> = [137 [/meteorite/i, 'meteorites'],138 [/fossil|ammonite|trilobite|dinosaur|mammoth|megalodon/i, 'fossils'],139 [/mineral|crystal|quartz|amethyst|geode|specimen/i, 'minerals'],140 [/wristwatch|pocket watch|chronograph/i, 'other_watches'],141 [/rolex/i, 'rolex'],142 [/patek philippe/i, 'patek_philippe'],143 [/omega/i, 'omega'],144];145export function categoryForLot(auctionCategory: string, title: string): string {146 return LOT_CATEGORY.find(([re]) => re.test(title))?.[1] ?? auctionCategory;147}148149/** Buyer's premium for a hammer price from tiered percentages (each tier applies to the slice above `amount_over`). */150export function premiumFor(hammer: number, tiers: Array<{ percent: number; amount_over: number }>): number | null {151 if (!tiers.length) return null;152 const sorted = [...tiers].sort((a, b) => a.amount_over - b.amount_over);153 let total = 0;154 for (let i = 0; i < sorted.length; i++) {155 const from = sorted[i]!.amount_over;156 const to = sorted[i + 1]?.amount_over ?? Infinity;157 if (hammer <= from) break;158 total += (Math.min(hammer, to) - from) * (sorted[i]!.percent / 100);159 }160 return Math.round(total * 100) / 100;161}162163export class LyonTurnbullConnector extends BaseConnector {164 readonly version = '1.0.0';165 readonly parserVersion = PARSER_VERSION;166 protected override minIntervalMs = 2000;167168 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {169 const pastUrl = String(this.meta.config.pastAuctionsUrl ?? `${BASE}/auctions/past-auctions`);170 const perRun = Number(this.meta.config.auctionsPerRun ?? 3);171 const cursor = (ctx.options.cursor ?? {}) as { done?: string[] };172 const done = new Set<string>(cursor.done ?? []);173 const list = await ctx.fetch(pastUrl, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });174 if (!list.success || !list.html) {175 ctx.anomaly('past_list_failed', list.error ?? String(list.httpStatus));176 return;177 }178 const auctions = parsePastAuctions(list.html).sort((a, b) => (b.amsDate ?? '').localeCompare(a.amsDate ?? ''));179 let fetched = 0;180 let count = 0;181 for (const auction of auctions) {182 if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun) break;183 if (done.has(auction.saleNumber) && ctx.options.mode !== 'backfill') continue;184 const url = BASE + auction.link;185 await this.throttle();186 fetched++;187 const res = await ctx.fetch(url, {188 engines: ['api'],189 responseType: 'text',190 timeoutMs: 60_000,191 expect: ['title', 'price', 'currency', 'status'],192 parse: (r) => {193 const p = r.html ? parseAuctionPage(r.html, url, auction) : null;194 const sold = p?.lots.find((l) => l.hammer);195 return p ? { title: p.lots[0]?.title, price: sold?.hammer ?? null, currency: sold ? p.currency : null, status: p.status } : null;196 },197 });198 const payload = res.success && res.html ? parseAuctionPage(res.html, url, auction) : null;199 if (!payload) {200 ctx.anomaly('auction_parse_failed', `${auction.saleNumber}: ${res.error ?? res.httpStatus}`);201 continue;202 }203 if (payload.status && payload.status !== 'completed') {204 // still live/published — results not final yet205 continue;206 }207 count++;208 done.add(auction.saleNumber);209 await ctx.setCursor({ done: [...done].slice(-400) });210 yield { url, externalId: auction.saleNumber, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };211 }212 }213214 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {215 const p = AuctionPayloadSchema.parse(raw.payload);216 const categorySlug = categoryForAuction(`${p.auction.title} ${p.auction.subtitle ?? ''}`);217 const out: NormalizedRecord[] = [];218 for (const lot of p.lots) {219 const saleDateIso = lot.sessionDate ?? p.auction.amsDate;220 const saleDate = saleDateIso ? new Date(saleDateIso) : null;221 const lotUrl = `${BASE}${p.auction.link}/lot/${lot.lotNo}`;222 const yearHint = lot.subTitle ? extractYear(lot.subTitle) : null;223 const attributes = AssetAttributesSchema.parse({224 categorySlug: categoryForLot(categorySlug, `${lot.title} ${lot.subTitle ?? ''}`),225 name: lot.title,226 year: yearHint,227 identifiers: { lt_lot_uuid: lot.uuid, lt_lot: `${p.auction.saleNumber}/${lot.lotNo}` },228 metadata: { sale_number: p.auction.saleNumber, auction_title: p.auction.title, sub_title: lot.subTitle, estimate_low: lot.low, estimate_high: lot.high, premium_tiers: p.premiumTiers, premium_estimate: lot.hammer ? premiumFor(lot.hammer, p.premiumTiers) : null },229 });230 const base = {231 connectorId: this.meta.id,232 sourceId: this.meta.sourceId,233 sourceUrl: lotUrl,234 externalId: `${p.auction.saleNumber}:${lot.lotNo}`,235 rawTitle: lot.subTitle ? `${lot.title} — ${lot.subTitle}` : lot.title,236 description: null,237 imageUrls: lot.image ? [lot.image] : [],238 attributes,239 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },240 condition: { condition: null, conditionRaw: null, completeness: null },241 observedAt: raw.fetchedAt,242 confidence: 0.9,243 parserVersion: PARSER_VERSION,244 };245 if (lot.hammer && saleDate && lot.status !== 'unsold') {246 out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate, price: lot.hammer, currency: p.currency, buyerPremiumIncluded: false, quantity: 1, isBundle: false, location: 'United Kingdom', auctionHouse: 'Lyon & Turnbull', lotNumber: lot.lotNo }));247 } else {248 out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: 'Lyon & Turnbull', auctionName: p.auction.title, lotNumber: lot.lotNo, startsAt: saleDate, endsAt: saleDate, estimateLow: lot.low, estimateHigh: lot.high, currentBid: null, currency: p.currency, status: 'ended', location: 'United Kingdom' }));249 }250 }251 return out;252 }253}254255export default function createConnector(meta: ConnectorMeta) {256 return new LyonTurnbullConnector(meta);257}258