TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { CurrencyCode, NormalizedRecord } from '@rareindex/shared';4import { makeSale, vehicleAttributes } from '../../firecrawl/_carlib/index.js';56/**7 * Gooding & Company prices realized. The Gatsby site serves every realized auction's lot list as8 * static JSON (page-data); one raw record per auction (compact lots), one sale per lot with a price.9 */10const BASE = 'https://www.goodingco.com';11const CLOUDINARY = 'https://res.cloudinary.com/goodingco/image/upload/c_fill,w_1200';12const PARSER_VERSION = '1.0.0';1314export const LotSchema = z.object({15 slug: z.string(),16 lotNumber: z.union([z.number(), z.string()]).nullable(),17 salePrice: z.number().nullable(),18 privateSalesPrice: z.boolean().nullable().optional(),19 title: z.string(),20 modelYear: z.number().nullable().optional(),21 make: z.string().nullable().optional(),22 model: z.string().nullable().optional(),23 itemType: z.string().nullable().optional(),24 image: z.string().nullable().optional(),25});26export type Lot = z.infer<typeof LotSchema>;27export const AuctionPayloadSchema = z.object({28 kind: z.literal('realized_auction'),29 slug: z.string(),30 url: z.string(),31 name: z.string(),32 currency: z.string(),33 saleDate: z.string().nullable(),34 sellThroughRate: z.number().nullable(),35 lots: z.array(LotSchema),36});37export type AuctionPayload = z.infer<typeof AuctionPayloadSchema>;3839interface PageData {40 result?: { data?: { contentfulWebPageAuction?: { title?: string; auction?: RawAuction }; contentfulLot?: RawLot & { auction?: RawAuction } } };41}42interface RawAuction {43 name?: string;44 currency?: string;45 sellThroughRate?: number | null;46 subEvents?: Array<{ __typename?: string; startDate?: string; endDate?: string }>;47 lot?: RawLot[];48}49interface RawLot {50 slug?: string;51 lotNumber?: number | string | null;52 salePrice?: number | null;53 privateSalesPrice?: boolean | null;54 item?: { __typename?: string; title?: string; modelYear?: number | null; make?: { name?: string } | null; model?: string | null; cloudinaryImagesCombined?: Array<{ public_id?: string }> | null } | null;55}5657export function imageUrl(publicId: string | undefined | null): string | null {58 if (!publicId) return null;59 return `${CLOUDINARY}/${publicId.split('/').map(encodeURIComponent).join('/')}`;60}6162/** Sale date = end of the last auction sub-event (calendar day in the venue's offset, stored as UTC midnight). */63export function auctionSaleDate(a: RawAuction | undefined): string | null {64 const days = (a?.subEvents ?? []).filter((s) => s.__typename === 'ContentfulSubEventAuction' && (s.endDate || s.startDate)).map((s) => (s.endDate ?? s.startDate)!.slice(0, 10));65 if (!days.length) return null;66 const last = days.sort().at(-1)!;67 return `${last}T00:00:00.000Z`;68}6970export function trimLot(l: RawLot): Lot | null {71 if (!l.slug || !l.item?.title) return null;72 return LotSchema.parse({73 slug: l.slug,74 lotNumber: l.lotNumber ?? null,75 salePrice: typeof l.salePrice === 'number' ? l.salePrice : null,76 privateSalesPrice: l.privateSalesPrice ?? null,77 title: l.item.title,78 modelYear: l.item.modelYear ?? null,79 make: l.item.make?.name ?? null,80 model: l.item.model ?? null,81 itemType: l.item.__typename ?? null,82 image: imageUrl(l.item.cloudinaryImagesCombined?.[0]?.public_id),83 });84}8586export function parseAuctionPageData(json: unknown, slug: string): AuctionPayload | null {87 const page = (json as PageData)?.result?.data?.contentfulWebPageAuction;88 const a = page?.auction;89 if (!a) return null;90 const lots = (a.lot ?? []).map(trimLot).filter((x): x is Lot => Boolean(x));91 return { kind: 'realized_auction', slug, url: `${BASE}/auction/realized/${slug}`, name: a.name ?? page?.title ?? slug, currency: (a.currency ?? 'USD').toUpperCase(), saleDate: auctionSaleDate(a), sellThroughRate: a.sellThroughRate ?? null, lots };92}9394export function discoverRealizedSlugs(html: string): string[] {95 return [...new Set([...html.matchAll(/\/auction\/realized\/([a-z0-9-]+)/g)].map((m) => m[1]!))];96}9798export class GoodingConnector extends BaseConnector {99 readonly version = '1.0.0';100 readonly parserVersion = PARSER_VERSION;101 protected override minIntervalMs = 1500;102 override readonly urlPatterns = [/^https?:\/\/www\.goodingco\.com\/lot\/[a-z0-9-]+\/?$/i];103104 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {105 const seeds = (this.meta.config.seeds as string[] | undefined) ?? [];106 const perRun = Number(this.meta.config.auctionsPerRun ?? 3);107 const done = new Set<string>((ctx.options.cursor?.done as string[] | undefined) ?? []);108 const backfill = ctx.options.mode === 'backfill';109 // Discover currently linked realized auctions from the homepage (plain HTML).110 let discovered: string[] = [];111 const home = await ctx.fetch(`${BASE}/`, { engines: ['api'], responseType: 'text', minQuality: 0 });112 if (home.success && home.html) discovered = discoverRealizedSlugs(home.html);113 const slugs = [...new Set([...discovered, ...seeds])];114 // Incremental: newest first, skip auctions already ingested unless backfilling.115 const todo = slugs.filter((s) => backfill || !done.has(s)).slice(0, perRun);116 let count = 0;117 for (const slug of todo) {118 if (ctx.signal?.aborted || this.reached(ctx, count)) break;119 const url = `${BASE}/page-data/auction/realized/${slug}/page-data.json`;120 await this.throttle();121 const res = await ctx.fetch(url, {122 engines: ['api'],123 expect: ['title', 'price', 'currency', 'date'],124 parse: (r) => {125 const p = parseAuctionPageData(r.json, slug);126 const sold = p?.lots.find((l) => l.salePrice);127 return p ? { title: p.name, price: sold?.salePrice ?? null, currency: p.currency, date: p.saleDate } : null;128 },129 });130 const payload = res.success ? parseAuctionPageData(res.json, slug) : null;131 if (!payload) {132 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);133 continue;134 }135 if (!payload.lots.length) ctx.anomaly('empty_page', url);136 count++;137 yield { url: payload.url, externalId: `auction:${slug}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };138 done.add(slug);139 await ctx.setCursor({ done: [...done], updatedAt: new Date().toISOString() });140 }141 }142143 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {144 const slug = url.match(/\/lot\/([a-z0-9-]+)/i)?.[1];145 if (!slug) return [];146 const res = await ctx.fetch(`${BASE}/page-data/lot/${slug}/page-data.json`, { engines: ['api'], minQuality: 0 });147 const lot = (res.json as PageData)?.result?.data?.contentfulLot;148 if (!res.success || !lot?.auction) return [];149 const item = trimLot({ ...lot, slug });150 if (!item) return [];151 const payload: AuctionPayload = { kind: 'realized_auction', slug: `lot-${slug}`, url, name: lot.auction.name ?? '', currency: (lot.auction.currency ?? 'USD').toUpperCase(), saleDate: auctionSaleDate(lot.auction), sellThroughRate: null, lots: [item] };152 return [{ url, externalId: `lot:${slug}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];153 }154155 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {156 const p = AuctionPayloadSchema.parse(raw.payload);157 if (!p.saleDate) return [];158 const saleDate = new Date(p.saleDate);159 const currency = p.currency as CurrencyCode;160 const out: NormalizedRecord[] = [];161 for (const lot of p.lots) {162 if (!lot.salePrice || lot.salePrice <= 0 || lot.privateSalesPrice) continue;163 const isVehicle = lot.itemType !== 'ContentfulAutomobilia' && Boolean(lot.make || lot.modelYear);164 const attributes = vehicleAttributes(lot.title, {165 ...(isVehicle ? {} : { categorySlug: 'automotive_memorabilia' }),166 identifiers: { gooding_lot: lot.slug },167 metadata: { auction: p.name, lot_number: lot.lotNumber, make_field: lot.make, model_field: lot.model, model_year_field: lot.modelYear, sell_through_rate: p.sellThroughRate },168 });169 if (lot.make) attributes.brand = lot.make;170 if (lot.modelYear) attributes.year = lot.modelYear;171 if (lot.model) attributes.model = lot.model;172 out.push(173 makeSale({174 meta: this.meta,175 sourceUrl: `${BASE}/lot/${lot.slug}`,176 externalId: lot.slug,177 rawTitle: lot.title,178 attributes,179 price: lot.salePrice,180 currency,181 saleDate,182 buyerPremiumIncluded: null,183 auctionHouse: 'Gooding & Company',184 lotNumber: lot.lotNumber === null ? null : String(lot.lotNumber),185 imageUrls: lot.image ? [lot.image] : [],186 observedAt: raw.fetchedAt,187 parserVersion: PARSER_VERSION,188 }),189 );190 }191 return out;192 }193}194195export default function createConnector(meta: ConnectorMeta): GoodingConnector {196 return new GoodingConnector(meta);197}198