import { z } from 'zod'; import { BaseConnector, adapters, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord } from '@rareindex/shared'; import { hintFromLabel, slugFromTitle, watchBrand } from '../_auction-lib/categories.js'; import { amount, houseCategory, inertiaPage, isBundleTitle, isoDate, lotAttributes, makeLot, makeSale, safeYear, saleGrade } from '../_g7-auctions-na-lib/index.js'; const PARSER_VERSION = '1.0.0'; export interface RagoSite { site: string; house: string; idKey: string; } export const WRIGHT: RagoSite = { site: 'https://www.wright20.com', house: 'Wright', idKey: 'wright_item_id' }; export const ItemSchema = z.object({ id: z.string(), alias: z.string(), artist: z.string().nullable(), name: z.string(), lotNumber: z.string().nullable(), estimateLow: z.number().nullable(), estimateHigh: z.number().nullable(), /** price realized as displayed by the house (null when unsold / not shown) */ result: z.number().nullable(), bidCount: z.number().nullable(), image: z.string().nullable(), location: z.string().nullable(), isRecord: z.boolean(), buyNow: z.boolean(), startingBid: z.number().nullable(), }); export const SessionSchema = z.object({ id: z.string(), alias: z.string(), title: z.string(), auctionTitle: z.string().nullable(), /** sale start (auction.date ISO) */ date: z.string().nullable(), startDate: z.number().nullable(), /** actual close time from the /info page when fetched */ endedAt: z.string().nullable(), saleStatus: z.string().nullable(), showResults: z.boolean(), /** null when the /info page (style.show_results_with_premium) was not available */ resultsIncludePremium: z.boolean().nullable(), preliminary: z.boolean(), archive: z.boolean(), timezone: z.string().nullable(), }); export const PayloadSchema = z.object({ kind: z.literal('wright_session'), site: z.string(), house: z.string(), session: SessionSchema, items: z.array(ItemSchema) }); export type Payload = z.infer; type InertiaProps = { auctions?: Record; sessions?: Record; items?: Record; session?: { fd_key?: number | string; results_are_preliminary?: number | boolean | null; archive_auction?: boolean | number | null; sale_status?: string | null }; modules?: Array<{ data?: { item?: { session?: { ended_at?: string | null; style?: { show_results_with_premium?: number | boolean | null; show_results?: number | boolean | null } } } } }>; }; type RawItem = { id?: number | string; fd_key?: number | string; alias?: string; artist_name?: string | null; name?: string | null; lot_number?: number | string | null; estimate_low?: number | null; estimate_high?: number | null; result?: number | boolean | null; bid_count?: number | null; primary_index_image?: string | null; location?: string | null; is_record_result?: number | boolean | null; buy_now?: number | boolean | null; starting_bid?: string | number | null }; function truthy(v: unknown): boolean { return v === true || v === 1 || v === '1'; } /** Session URL derived from a sitemap-index child name: sitemap-auctions_2026_08_design.xml → /auctions/2026/08/design */ export function sessionUrlFromSitemap(site: string, loc: string): string | null { const m = loc.match(/sitemap-auctions_(\d{4})_(\d{2})_([a-z0-9-]+)\.xml$/i); return m ? `${site}/auctions/${m[1]}/${m[2]}/${m[3]}` : null; } /** Info-page payload (`/auctions/yyyy/mm/slug/info`) → close time and premium flag when present. */ export function parseInfoPage(html: string): { endedAt: string | null; resultsIncludePremium: boolean | null } { const page = inertiaPage<{ props?: InertiaProps }>(html); const mods = page?.props?.modules ?? []; for (const m of mods) { const s = m?.data?.item?.session; if (!s) continue; const prem = s.style?.show_results_with_premium; return { endedAt: s.ended_at ?? null, resultsIncludePremium: prem === null || prem === undefined ? null : truthy(prem) }; } return { endedAt: null, resultsIncludePremium: null }; } /** Session index page → session header + all lots (the page carries the full session; no pager). */ export function parseSessionPage(html: string, cfg: RagoSite, pageUrl: string): Payload | null { const page = inertiaPage<{ component?: string; props?: InertiaProps }>(html); const p = page?.props; if (!p || !p.items) return null; const sessions = Object.values(p.sessions ?? {}); const s = sessions.find((x) => String(x.fd_key) === String(p.session?.fd_key)) ?? sessions[0]; const auction = Object.values(p.auctions ?? {})[0]; const alias = s?.alias ?? new URL(pageUrl).pathname; const session: z.infer = { id: String(s?.fd_key ?? p.session?.fd_key ?? alias), alias: alias.startsWith('/') ? alias : `/${alias}`, title: s?.title ?? auction?.title ?? '', auctionTitle: auction?.title ?? null, date: auction?.date ?? null, startDate: s?.start_date !== undefined && s?.start_date !== null ? Number(s.start_date) : null, endedAt: null, saleStatus: s?.sale_status ?? p.session?.sale_status ?? null, showResults: truthy(s?.show_results), resultsIncludePremium: null, preliminary: truthy(s?.results_are_preliminary ?? p.session?.results_are_preliminary), archive: truthy(s?.archive_auction ?? p.session?.archive_auction), timezone: s?.timezone ?? null, }; const items: Payload['items'] = []; for (const it of Object.values(p.items)) { const id = it.id ?? it.fd_key; if (id === undefined || !it.alias || !it.name) continue; items.push({ id: String(id), alias: it.alias.replace(/^\//, ''), artist: it.artist_name?.trim() || null, name: it.name.trim(), lotNumber: it.lot_number !== null && it.lot_number !== undefined ? String(it.lot_number) : null, estimateLow: amount(it.estimate_low), estimateHigh: amount(it.estimate_high), result: typeof it.result === 'number' ? amount(it.result) : null, bidCount: typeof it.bid_count === 'number' ? it.bid_count : null, image: it.primary_index_image ?? null, location: it.location ?? null, isRecord: truthy(it.is_record_result), buyNow: truthy(it.buy_now), startingBid: amount(it.starting_bid), }); } return { kind: 'wright_session', site: cfg.site, house: cfg.house, session, items }; } /** Sale-title driven mapping; Wright/LAMA sell design, art, prints, photographs, jewelry and tribal art. */ export function ragoCategory(sessionTitle: string, title: string): string | null { const s = sessionTitle.toLowerCase(); if (/africa|oceania|americas|tribal|pre-columbian/.test(s)) return 'antiques'; if (/photograph/.test(s)) return 'photography'; if (/jewel|watch/.test(s)) return /\b(watch|wristwatch|chronograph)\b/i.test(title) ? watchBrand(title).slug : slugFromTitle(title, 'jewelry') ?? 'jewelry'; if (/design|furniture|lighting|ceramics|glass|studio|nakashima|eames|scandinavian|italian|american/.test(s)) { // Design sales mix furniture with art/prints/photographs: trust a confident title match, else design. const t = slugFromTitle(title, 'unknown'); return t && t !== 'antiques' ? t : 'design_furniture'; } if (/print|edition|multiple|murakami|kaws|works on paper|contemporary|post-war|modern art|art\b|paintings|sculpture|abstract/.test(s)) return slugFromTitle(title, hintFromLabel(sessionTitle) === 'unknown' ? 'art' : hintFromLabel(sessionTitle)) ?? 'art'; if (/book|rare/.test(s)) return slugFromTitle(title, 'books') ?? 'books'; if (/poster/.test(s)) return 'movie_posters'; return houseCategory(sessionTitle, title, null); } /** * Rago/Wright group platform (wright20.com, lamodern.com). Session index pages are server-rendered with an * Inertia `data-page` JSON that holds every lot of the sale (estimate, result). See meta.json accessNotes. */ export class RagoWrightConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 10_000; // robots.txt Crawl-delay: 10 constructor(meta: ConnectorMeta, protected readonly cfg: RagoSite) { super(meta); } private async html(ctx: CrawlContext, url: string): Promise<{ html: string | null; status: number | null; fetchedAt: Date }> { await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); return { html: null, status: res.httpStatus, fetchedAt: res.fetchedAt }; } return { html: res.html, status: res.httpStatus, fetchedAt: res.fetchedAt }; } /** Session URLs, newest first, from the sitemap index (children are never fetched). */ protected async listSessions(ctx: CrawlContext): Promise { if (ctx.options.seeds?.length) return ctx.options.seeds.map((s) => s.replace(/\/(info|\d+)\/?$/, '').replace(/\/$/, '')); const url = `${this.cfg.site}/sitemap-index.xml`; await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, force: true }); if (!res.success || !res.html) { ctx.anomaly('sitemap_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); return []; } const out: string[] = []; for (const e of adapters.parseSitemapIndex(res.html)) { const u = sessionUrlFromSitemap(this.cfg.site, e.loc); if (u && !out.includes(u)) out.push(u); } if (!out.length) ctx.anomaly('pagination_failure', 'sitemap index listed no session sitemaps'); return out; } async *crawl(ctx: CrawlContext): AsyncIterable { const perRun = ctx.options.mode === 'probe' ? 1 : Number(this.meta.config.sessionsPerRun ?? 2); const fetchInfo = this.meta.config.fetchInfoPage !== false && ctx.options.mode !== 'probe'; const cursor = ctx.options.cursor ?? {}; const doneSessions = new Set(Array.isArray(cursor.doneSessions) ? (cursor.doneSessions as string[]) : []); const sessions = await this.listSessions(ctx); if (!sessions.length) return; const backfill = ctx.options.mode === 'backfill'; // Backfill walks oldest → newest from the saved index; incremental takes the newest undone sessions. let order: string[]; let backfillIndex = Number(cursor.backfillIndex ?? 0); if (backfill) order = [...sessions].reverse().slice(backfillIndex); else { const recent = new Set(sessions.slice(0, 3)); // newest sessions are re-read until they close order = sessions.filter((u) => !doneSessions.has(u) || recent.has(u)); } let processed = 0; let count = 0; let items = 0; const now = Date.now(); for (const url of order) { if (ctx.signal?.aborted || processed >= perRun || this.reached(ctx, count)) break; const r = await this.html(ctx, url); processed++; if (!r.html) { if (backfill) backfillIndex++; continue; } const payload = parseSessionPage(r.html, this.cfg, url); if (!payload) { ctx.anomaly('parse_failure_page', `${url}: no Inertia data-page payload`); if (backfill) backfillIndex++; continue; } if (fetchInfo && payload.items.some((i) => i.result !== null)) { const info = await this.html(ctx, `${url}/info`); if (info.html) { const parsed = parseInfoPage(info.html); payload.session.endedAt = parsed.endedAt; payload.session.resultsIncludePremium = parsed.resultsIncludePremium; } } count++; items += payload.items.length; const start = payload.session.startDate ? payload.session.startDate * 1000 : isoDate(payload.session.date)?.getTime() ?? 0; const closed = start > 0 && start < now - 2 * 86_400_000 && payload.items.some((i) => i.result !== null); yield { url, externalId: `session:${payload.session.id}`, kind: closed ? 'sale' : 'auction_lot', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt }; if (closed) doneSessions.add(url); if (backfill) { backfillIndex++; await ctx.progress({ page: backfillIndex, totalPages: sessions.length, itemsProcessed: items, reachedDate: payload.session.date ? isoDate(payload.session.date) : null }); } await ctx.setCursor({ doneSessions: [...doneSessions].slice(-800), backfillIndex, updatedAt: new Date().toISOString() }); } if (backfill && backfillIndex >= sessions.length) await ctx.setCursor({ doneSessions: [...doneSessions].slice(-800), backfillIndex, done: true, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const s = p.session; const start = s.startDate ? new Date(s.startDate * 1000) : isoDate(s.date); const saleDate = isoDate(s.endedAt) ?? isoDate(s.date) ?? start; const now = Date.now(); const upcoming = start !== null && start.getTime() > now; let unmapped = 0; for (const it of p.items) { const fullTitle = it.artist ? `${it.artist}, ${it.name}` : it.name; const slug = ragoCategory(s.title || s.auctionTitle || '', fullTitle); if (!slug) { unmapped++; continue; } const g = saleGrade(fullTitle); const attributes = lotAttributes({ categorySlug: slug, name: it.name, year: safeYear(it.name), identifiers: { [this.cfg.idKey]: it.id }, metadata: { artist: it.artist, session_id: s.id, session_title: s.title, auction_title: s.auctionTitle, estimate_low: it.estimateLow, estimate_high: it.estimateHigh, bid_count: it.bidCount, location: it.location, is_record_result: it.isRecord, ...(s.preliminary ? { preliminary: true } : {}) } }); const common = { meta: this.meta, sourceUrl: `${p.site}/${it.alias}`, externalId: it.id, rawTitle: fullTitle, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: it.location ?? 'US' }; if (it.result !== null && s.showResults && !upcoming) { if (!saleDate || saleDate.getTime() > now + 86_400_000) continue; out.push(makeSale({ ...common, price: it.result, currency: 'USD', saleDate, buyerPremiumIncluded: s.resultsIncludePremium, auctionHouse: p.house, lotNumber: it.lotNumber, isBundle: isBundleTitle(fullTitle), confidence: s.preliminary ? 0.75 : 0.88 })); } else if (upcoming || (it.result === null && !s.archive && s.saleStatus !== 'POSTSALE')) { out.push(makeLot({ ...common, auctionHouse: p.house, auctionName: s.auctionTitle ?? s.title, lotNumber: it.lotNumber, startsAt: start, endsAt: null, estimateLow: it.estimateLow, estimateHigh: it.estimateHigh, currentBid: null, currency: 'USD', status: upcoming ? 'upcoming' : 'live', confidence: 0.8 })); } } return out; } } export default (meta: ConnectorMeta) => new RagoWrightConnector(meta, WRIGHT);