TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, adapters, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { hintFromLabel, slugFromTitle, watchBrand } from '../_auction-lib/categories.js';5import { amount, houseCategory, inertiaPage, isBundleTitle, isoDate, lotAttributes, makeLot, makeSale, safeYear, saleGrade } from '../_g7-auctions-na-lib/index.js';67const PARSER_VERSION = '1.0.0';89export interface RagoSite {10 site: string;11 house: string;12 idKey: string;13}14export const WRIGHT: RagoSite = { site: 'https://www.wright20.com', house: 'Wright', idKey: 'wright_item_id' };1516export const ItemSchema = z.object({17 id: z.string(),18 alias: z.string(),19 artist: z.string().nullable(),20 name: z.string(),21 lotNumber: z.string().nullable(),22 estimateLow: z.number().nullable(),23 estimateHigh: z.number().nullable(),24 /** price realized as displayed by the house (null when unsold / not shown) */25 result: z.number().nullable(),26 bidCount: z.number().nullable(),27 image: z.string().nullable(),28 location: z.string().nullable(),29 isRecord: z.boolean(),30 buyNow: z.boolean(),31 startingBid: z.number().nullable(),32});33export const SessionSchema = z.object({34 id: z.string(),35 alias: z.string(),36 title: z.string(),37 auctionTitle: z.string().nullable(),38 /** sale start (auction.date ISO) */39 date: z.string().nullable(),40 startDate: z.number().nullable(),41 /** actual close time from the /info page when fetched */42 endedAt: z.string().nullable(),43 saleStatus: z.string().nullable(),44 showResults: z.boolean(),45 /** null when the /info page (style.show_results_with_premium) was not available */46 resultsIncludePremium: z.boolean().nullable(),47 preliminary: z.boolean(),48 archive: z.boolean(),49 timezone: z.string().nullable(),50});51export const PayloadSchema = z.object({ kind: z.literal('wright_session'), site: z.string(), house: z.string(), session: SessionSchema, items: z.array(ItemSchema) });52export type Payload = z.infer<typeof PayloadSchema>;5354type InertiaProps = {55 auctions?: Record<string, { title?: string; date?: string | null; auction_house?: number | string }>;56 sessions?: Record<string, { fd_key?: number | string; alias?: string; title?: string; start_date?: number | string | null; sale_status?: string | null; show_results?: number | boolean | null; results_are_preliminary?: number | boolean | null; archive_auction?: number | boolean | null; timezone?: string | null }>;57 items?: Record<string, RawItem>;58 session?: { fd_key?: number | string; results_are_preliminary?: number | boolean | null; archive_auction?: boolean | number | null; sale_status?: string | null };59 modules?: Array<{ data?: { item?: { session?: { ended_at?: string | null; style?: { show_results_with_premium?: number | boolean | null; show_results?: number | boolean | null } } } } }>;60};61type RawItem = { id?: number | string; fd_key?: number | string; alias?: string; artist_name?: string | null; name?: string | null; lot_number?: number | string | null; estimate_low?: number | null; estimate_high?: number | null; result?: number | boolean | null; bid_count?: number | null; primary_index_image?: string | null; location?: string | null; is_record_result?: number | boolean | null; buy_now?: number | boolean | null; starting_bid?: string | number | null };6263function truthy(v: unknown): boolean {64 return v === true || v === 1 || v === '1';65}6667/** Session URL derived from a sitemap-index child name: sitemap-auctions_2026_08_design.xml → /auctions/2026/08/design */68export function sessionUrlFromSitemap(site: string, loc: string): string | null {69 const m = loc.match(/sitemap-auctions_(\d{4})_(\d{2})_([a-z0-9-]+)\.xml$/i);70 return m ? `${site}/auctions/${m[1]}/${m[2]}/${m[3]}` : null;71}7273/** Info-page payload (`/auctions/yyyy/mm/slug/info`) → close time and premium flag when present. */74export function parseInfoPage(html: string): { endedAt: string | null; resultsIncludePremium: boolean | null } {75 const page = inertiaPage<{ props?: InertiaProps }>(html);76 const mods = page?.props?.modules ?? [];77 for (const m of mods) {78 const s = m?.data?.item?.session;79 if (!s) continue;80 const prem = s.style?.show_results_with_premium;81 return { endedAt: s.ended_at ?? null, resultsIncludePremium: prem === null || prem === undefined ? null : truthy(prem) };82 }83 return { endedAt: null, resultsIncludePremium: null };84}8586/** Session index page → session header + all lots (the page carries the full session; no pager). */87export function parseSessionPage(html: string, cfg: RagoSite, pageUrl: string): Payload | null {88 const page = inertiaPage<{ component?: string; props?: InertiaProps }>(html);89 const p = page?.props;90 if (!p || !p.items) return null;91 const sessions = Object.values(p.sessions ?? {});92 const s = sessions.find((x) => String(x.fd_key) === String(p.session?.fd_key)) ?? sessions[0];93 const auction = Object.values(p.auctions ?? {})[0];94 const alias = s?.alias ?? new URL(pageUrl).pathname;95 const session: z.infer<typeof SessionSchema> = {96 id: String(s?.fd_key ?? p.session?.fd_key ?? alias),97 alias: alias.startsWith('/') ? alias : `/${alias}`,98 title: s?.title ?? auction?.title ?? '',99 auctionTitle: auction?.title ?? null,100 date: auction?.date ?? null,101 startDate: s?.start_date !== undefined && s?.start_date !== null ? Number(s.start_date) : null,102 endedAt: null,103 saleStatus: s?.sale_status ?? p.session?.sale_status ?? null,104 showResults: truthy(s?.show_results),105 resultsIncludePremium: null,106 preliminary: truthy(s?.results_are_preliminary ?? p.session?.results_are_preliminary),107 archive: truthy(s?.archive_auction ?? p.session?.archive_auction),108 timezone: s?.timezone ?? null,109 };110 const items: Payload['items'] = [];111 for (const it of Object.values(p.items)) {112 const id = it.id ?? it.fd_key;113 if (id === undefined || !it.alias || !it.name) continue;114 items.push({115 id: String(id),116 alias: it.alias.replace(/^\//, ''),117 artist: it.artist_name?.trim() || null,118 name: it.name.trim(),119 lotNumber: it.lot_number !== null && it.lot_number !== undefined ? String(it.lot_number) : null,120 estimateLow: amount(it.estimate_low),121 estimateHigh: amount(it.estimate_high),122 result: typeof it.result === 'number' ? amount(it.result) : null,123 bidCount: typeof it.bid_count === 'number' ? it.bid_count : null,124 image: it.primary_index_image ?? null,125 location: it.location ?? null,126 isRecord: truthy(it.is_record_result),127 buyNow: truthy(it.buy_now),128 startingBid: amount(it.starting_bid),129 });130 }131 return { kind: 'wright_session', site: cfg.site, house: cfg.house, session, items };132}133134/** Sale-title driven mapping; Wright/LAMA sell design, art, prints, photographs, jewelry and tribal art. */135export function ragoCategory(sessionTitle: string, title: string): string | null {136 const s = sessionTitle.toLowerCase();137 if (/africa|oceania|americas|tribal|pre-columbian/.test(s)) return 'antiques';138 if (/photograph/.test(s)) return 'photography';139 if (/jewel|watch/.test(s)) return /\b(watch|wristwatch|chronograph)\b/i.test(title) ? watchBrand(title).slug : slugFromTitle(title, 'jewelry') ?? 'jewelry';140 if (/design|furniture|lighting|ceramics|glass|studio|nakashima|eames|scandinavian|italian|american/.test(s)) {141 // Design sales mix furniture with art/prints/photographs: trust a confident title match, else design.142 const t = slugFromTitle(title, 'unknown');143 return t && t !== 'antiques' ? t : 'design_furniture';144 }145 if (/print|edition|multiple|murakami|kaws|works on paper|contemporary|post-war|modern art|art\b|paintings|sculpture|abstract/.test(s)) return slugFromTitle(title, hintFromLabel(sessionTitle) === 'unknown' ? 'art' : hintFromLabel(sessionTitle)) ?? 'art';146 if (/book|rare/.test(s)) return slugFromTitle(title, 'books') ?? 'books';147 if (/poster/.test(s)) return 'movie_posters';148 return houseCategory(sessionTitle, title, null);149}150151/**152 * Rago/Wright group platform (wright20.com, lamodern.com). Session index pages are server-rendered with an153 * Inertia `data-page` JSON that holds every lot of the sale (estimate, result). See meta.json accessNotes.154 */155export class RagoWrightConnector extends BaseConnector {156 readonly version = '1.0.0';157 readonly parserVersion = PARSER_VERSION;158 protected override minIntervalMs = 10_000; // robots.txt Crawl-delay: 10159160 constructor(meta: ConnectorMeta, protected readonly cfg: RagoSite) {161 super(meta);162 }163164 private async html(ctx: CrawlContext, url: string): Promise<{ html: string | null; status: number | null; fetchedAt: Date }> {165 await this.throttle(url);166 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });167 if (!res.success || !res.html) {168 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);169 return { html: null, status: res.httpStatus, fetchedAt: res.fetchedAt };170 }171 return { html: res.html, status: res.httpStatus, fetchedAt: res.fetchedAt };172 }173174 /** Session URLs, newest first, from the sitemap index (children are never fetched). */175 protected async listSessions(ctx: CrawlContext): Promise<string[]> {176 if (ctx.options.seeds?.length) return ctx.options.seeds.map((s) => s.replace(/\/(info|\d+)\/?$/, '').replace(/\/$/, ''));177 const url = `${this.cfg.site}/sitemap-index.xml`;178 await this.throttle(url);179 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, force: true });180 if (!res.success || !res.html) {181 ctx.anomaly('sitemap_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);182 return [];183 }184 const out: string[] = [];185 for (const e of adapters.parseSitemapIndex(res.html)) {186 const u = sessionUrlFromSitemap(this.cfg.site, e.loc);187 if (u && !out.includes(u)) out.push(u);188 }189 if (!out.length) ctx.anomaly('pagination_failure', 'sitemap index listed no session sitemaps');190 return out;191 }192193 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {194 const perRun = ctx.options.mode === 'probe' ? 1 : Number(this.meta.config.sessionsPerRun ?? 2);195 const fetchInfo = this.meta.config.fetchInfoPage !== false && ctx.options.mode !== 'probe';196 const cursor = ctx.options.cursor ?? {};197 const doneSessions = new Set<string>(Array.isArray(cursor.doneSessions) ? (cursor.doneSessions as string[]) : []);198 const sessions = await this.listSessions(ctx);199 if (!sessions.length) return;200 const backfill = ctx.options.mode === 'backfill';201 // Backfill walks oldest → newest from the saved index; incremental takes the newest undone sessions.202 let order: string[];203 let backfillIndex = Number(cursor.backfillIndex ?? 0);204 if (backfill) order = [...sessions].reverse().slice(backfillIndex);205 else {206 const recent = new Set(sessions.slice(0, 3)); // newest sessions are re-read until they close207 order = sessions.filter((u) => !doneSessions.has(u) || recent.has(u));208 }209 let processed = 0;210 let count = 0;211 let items = 0;212 const now = Date.now();213 for (const url of order) {214 if (ctx.signal?.aborted || processed >= perRun || this.reached(ctx, count)) break;215 const r = await this.html(ctx, url);216 processed++;217 if (!r.html) {218 if (backfill) backfillIndex++;219 continue;220 }221 const payload = parseSessionPage(r.html, this.cfg, url);222 if (!payload) {223 ctx.anomaly('parse_failure_page', `${url}: no Inertia data-page payload`);224 if (backfill) backfillIndex++;225 continue;226 }227 if (fetchInfo && payload.items.some((i) => i.result !== null)) {228 const info = await this.html(ctx, `${url}/info`);229 if (info.html) {230 const parsed = parseInfoPage(info.html);231 payload.session.endedAt = parsed.endedAt;232 payload.session.resultsIncludePremium = parsed.resultsIncludePremium;233 }234 }235 count++;236 items += payload.items.length;237 const start = payload.session.startDate ? payload.session.startDate * 1000 : isoDate(payload.session.date)?.getTime() ?? 0;238 const closed = start > 0 && start < now - 2 * 86_400_000 && payload.items.some((i) => i.result !== null);239 yield { url, externalId: `session:${payload.session.id}`, kind: closed ? 'sale' : 'auction_lot', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt };240 if (closed) doneSessions.add(url);241 if (backfill) {242 backfillIndex++;243 await ctx.progress({ page: backfillIndex, totalPages: sessions.length, itemsProcessed: items, reachedDate: payload.session.date ? isoDate(payload.session.date) : null });244 }245 await ctx.setCursor({ doneSessions: [...doneSessions].slice(-800), backfillIndex, updatedAt: new Date().toISOString() });246 }247 if (backfill && backfillIndex >= sessions.length) await ctx.setCursor({ doneSessions: [...doneSessions].slice(-800), backfillIndex, done: true, updatedAt: new Date().toISOString() });248 }249250 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {251 const p = PayloadSchema.parse(raw.payload);252 const out: NormalizedRecord[] = [];253 const s = p.session;254 const start = s.startDate ? new Date(s.startDate * 1000) : isoDate(s.date);255 const saleDate = isoDate(s.endedAt) ?? isoDate(s.date) ?? start;256 const now = Date.now();257 const upcoming = start !== null && start.getTime() > now;258 let unmapped = 0;259 for (const it of p.items) {260 const fullTitle = it.artist ? `${it.artist}, ${it.name}` : it.name;261 const slug = ragoCategory(s.title || s.auctionTitle || '', fullTitle);262 if (!slug) {263 unmapped++;264 continue;265 }266 const g = saleGrade(fullTitle);267 const attributes = lotAttributes({ categorySlug: slug, name: it.name, year: safeYear(it.name), identifiers: { [this.cfg.idKey]: it.id }, metadata: { artist: it.artist, session_id: s.id, session_title: s.title, auction_title: s.auctionTitle, estimate_low: it.estimateLow, estimate_high: it.estimateHigh, bid_count: it.bidCount, location: it.location, is_record_result: it.isRecord, ...(s.preliminary ? { preliminary: true } : {}) } });268 const common = { meta: this.meta, sourceUrl: `${p.site}/${it.alias}`, externalId: it.id, rawTitle: fullTitle, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: it.location ?? 'US' };269 if (it.result !== null && s.showResults && !upcoming) {270 if (!saleDate || saleDate.getTime() > now + 86_400_000) continue;271 out.push(makeSale({ ...common, price: it.result, currency: 'USD', saleDate, buyerPremiumIncluded: s.resultsIncludePremium, auctionHouse: p.house, lotNumber: it.lotNumber, isBundle: isBundleTitle(fullTitle), confidence: s.preliminary ? 0.75 : 0.88 }));272 } else if (upcoming || (it.result === null && !s.archive && s.saleStatus !== 'POSTSALE')) {273 out.push(makeLot({ ...common, auctionHouse: p.house, auctionName: s.auctionTitle ?? s.title, lotNumber: it.lotNumber, startsAt: start, endsAt: null, estimateLow: it.estimateLow, estimateHigh: it.estimateHigh, currentBid: null, currency: 'USD', status: upcoming ? 'upcoming' : 'live', confidence: 0.8 }));274 }275 }276 return out;277 }278}279280export default (meta: ConnectorMeta) => new RagoWrightConnector(meta, WRIGHT);281