import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { CurrencyCode, NormalizedRecord } from '@rareindex/shared'; import { makeSale, vehicleAttributes } from '../../firecrawl/_carlib/index.js'; /** * Gooding & Company prices realized. The Gatsby site serves every realized auction's lot list as * static JSON (page-data); one raw record per auction (compact lots), one sale per lot with a price. */ const BASE = 'https://www.goodingco.com'; const CLOUDINARY = 'https://res.cloudinary.com/goodingco/image/upload/c_fill,w_1200'; const PARSER_VERSION = '1.0.0'; export const LotSchema = z.object({ slug: z.string(), lotNumber: z.union([z.number(), z.string()]).nullable(), salePrice: z.number().nullable(), privateSalesPrice: z.boolean().nullable().optional(), title: z.string(), modelYear: z.number().nullable().optional(), make: z.string().nullable().optional(), model: z.string().nullable().optional(), itemType: z.string().nullable().optional(), image: z.string().nullable().optional(), }); export type Lot = z.infer; export const AuctionPayloadSchema = z.object({ kind: z.literal('realized_auction'), slug: z.string(), url: z.string(), name: z.string(), currency: z.string(), saleDate: z.string().nullable(), sellThroughRate: z.number().nullable(), lots: z.array(LotSchema), }); export type AuctionPayload = z.infer; interface PageData { result?: { data?: { contentfulWebPageAuction?: { title?: string; auction?: RawAuction }; contentfulLot?: RawLot & { auction?: RawAuction } } }; } interface RawAuction { name?: string; currency?: string; sellThroughRate?: number | null; subEvents?: Array<{ __typename?: string; startDate?: string; endDate?: string }>; lot?: RawLot[]; } interface RawLot { slug?: string; lotNumber?: number | string | null; salePrice?: number | null; privateSalesPrice?: boolean | null; item?: { __typename?: string; title?: string; modelYear?: number | null; make?: { name?: string } | null; model?: string | null; cloudinaryImagesCombined?: Array<{ public_id?: string }> | null } | null; } export function imageUrl(publicId: string | undefined | null): string | null { if (!publicId) return null; return `${CLOUDINARY}/${publicId.split('/').map(encodeURIComponent).join('/')}`; } /** Sale date = end of the last auction sub-event (calendar day in the venue's offset, stored as UTC midnight). */ export function auctionSaleDate(a: RawAuction | undefined): string | null { const days = (a?.subEvents ?? []).filter((s) => s.__typename === 'ContentfulSubEventAuction' && (s.endDate || s.startDate)).map((s) => (s.endDate ?? s.startDate)!.slice(0, 10)); if (!days.length) return null; const last = days.sort().at(-1)!; return `${last}T00:00:00.000Z`; } export function trimLot(l: RawLot): Lot | null { if (!l.slug || !l.item?.title) return null; return LotSchema.parse({ slug: l.slug, lotNumber: l.lotNumber ?? null, salePrice: typeof l.salePrice === 'number' ? l.salePrice : null, privateSalesPrice: l.privateSalesPrice ?? null, title: l.item.title, modelYear: l.item.modelYear ?? null, make: l.item.make?.name ?? null, model: l.item.model ?? null, itemType: l.item.__typename ?? null, image: imageUrl(l.item.cloudinaryImagesCombined?.[0]?.public_id), }); } export function parseAuctionPageData(json: unknown, slug: string): AuctionPayload | null { const page = (json as PageData)?.result?.data?.contentfulWebPageAuction; const a = page?.auction; if (!a) return null; const lots = (a.lot ?? []).map(trimLot).filter((x): x is Lot => Boolean(x)); return { kind: 'realized_auction', slug, url: `${BASE}/auction/realized/${slug}`, name: a.name ?? page?.title ?? slug, currency: (a.currency ?? 'USD').toUpperCase(), saleDate: auctionSaleDate(a), sellThroughRate: a.sellThroughRate ?? null, lots }; } export function discoverRealizedSlugs(html: string): string[] { return [...new Set([...html.matchAll(/\/auction\/realized\/([a-z0-9-]+)/g)].map((m) => m[1]!))]; } export class GoodingConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/www\.goodingco\.com\/lot\/[a-z0-9-]+\/?$/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (this.meta.config.seeds as string[] | undefined) ?? []; const perRun = Number(this.meta.config.auctionsPerRun ?? 3); const done = new Set((ctx.options.cursor?.done as string[] | undefined) ?? []); const backfill = ctx.options.mode === 'backfill'; // Discover currently linked realized auctions from the homepage (plain HTML). let discovered: string[] = []; const home = await ctx.fetch(`${BASE}/`, { engines: ['api'], responseType: 'text', minQuality: 0 }); if (home.success && home.html) discovered = discoverRealizedSlugs(home.html); const slugs = [...new Set([...discovered, ...seeds])]; // Incremental: newest first, skip auctions already ingested unless backfilling. const todo = slugs.filter((s) => backfill || !done.has(s)).slice(0, perRun); let count = 0; for (const slug of todo) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${BASE}/page-data/auction/realized/${slug}/page-data.json`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], expect: ['title', 'price', 'currency', 'date'], parse: (r) => { const p = parseAuctionPageData(r.json, slug); const sold = p?.lots.find((l) => l.salePrice); return p ? { title: p.name, price: sold?.salePrice ?? null, currency: p.currency, date: p.saleDate } : null; }, }); const payload = res.success ? parseAuctionPageData(res.json, slug) : null; if (!payload) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); continue; } if (!payload.lots.length) ctx.anomaly('empty_page', url); count++; yield { url: payload.url, externalId: `auction:${slug}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; done.add(slug); await ctx.setCursor({ done: [...done], updatedAt: new Date().toISOString() }); } } async lookup(url: string, ctx: CrawlContext): Promise { const slug = url.match(/\/lot\/([a-z0-9-]+)/i)?.[1]; if (!slug) return []; const res = await ctx.fetch(`${BASE}/page-data/lot/${slug}/page-data.json`, { engines: ['api'], minQuality: 0 }); const lot = (res.json as PageData)?.result?.data?.contentfulLot; if (!res.success || !lot?.auction) return []; const item = trimLot({ ...lot, slug }); if (!item) return []; const payload: AuctionPayload = { kind: 'realized_auction', slug: `lot-${slug}`, url, name: lot.auction.name ?? '', currency: (lot.auction.currency ?? 'USD').toUpperCase(), saleDate: auctionSaleDate(lot.auction), sellThroughRate: null, lots: [item] }; return [{ url, externalId: `lot:${slug}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = AuctionPayloadSchema.parse(raw.payload); if (!p.saleDate) return []; const saleDate = new Date(p.saleDate); const currency = p.currency as CurrencyCode; const out: NormalizedRecord[] = []; for (const lot of p.lots) { if (!lot.salePrice || lot.salePrice <= 0 || lot.privateSalesPrice) continue; const isVehicle = lot.itemType !== 'ContentfulAutomobilia' && Boolean(lot.make || lot.modelYear); const attributes = vehicleAttributes(lot.title, { ...(isVehicle ? {} : { categorySlug: 'automotive_memorabilia' }), identifiers: { gooding_lot: lot.slug }, metadata: { auction: p.name, lot_number: lot.lotNumber, make_field: lot.make, model_field: lot.model, model_year_field: lot.modelYear, sell_through_rate: p.sellThroughRate }, }); if (lot.make) attributes.brand = lot.make; if (lot.modelYear) attributes.year = lot.modelYear; if (lot.model) attributes.model = lot.model; out.push( makeSale({ meta: this.meta, sourceUrl: `${BASE}/lot/${lot.slug}`, externalId: lot.slug, rawTitle: lot.title, attributes, price: lot.salePrice, currency, saleDate, buyerPremiumIncluded: null, auctionHouse: 'Gooding & Company', lotNumber: lot.lotNumber === null ? null : String(lot.lotNumber), imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): GoodingConnector { return new GoodingConnector(meta); }