TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedSaleSchema, extractYear, parsePrice, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';5import { classifyGoldinTitle, parseCardAttributes, isBundleTitle } from './classify.js';6import { scrapflyRender, type ScrapflyRenderResult } from './scrapfly-render.js';78/**9 * Goldin — sold auction results (§110). Primary engine: Scrapfly (JS rendering + ASP).10 * Data source = the lots_v2 search XHR performed by Goldin's own public "Sold Items" grid.11 * See meta.json accessNotes for the access/legal rationale.12 */1314const IMAGE_CDN = 'https://d2tt46f3mh26nl.cloudfront.net/public/Lots';15const SITE = 'https://goldin.co';1617export const GoldinLotSchema = z.object({18 lot_id: z.string(),19 title: z.string(),20 meta_slug: z.string(),21 current_price: z.number().nullable().optional(),22 buyer_premium: z.number().nullable().optional(),23 end_timestamp: z.string().nullable().optional(),24 start_timestamp: z.string().nullable().optional(),25 status: z.string().nullable().optional(),26 auction_id: z.string().nullable().optional(),27 auction_type: z.string().nullable().optional(),28 lot_number: z.number().nullable().optional(),29 number_of_bids: z.number().nullable().optional(),30 primary_image_name: z.string().nullable().optional(),31});32export type GoldinLot = z.infer<typeof GoldinLotSchema>;3334export const SeedSchema = z.object({ path: z.string(), categorySlug: z.string(), sport: z.string().optional() });35export type Seed = z.infer<typeof SeedSchema>;3637/** Raw payload for one rendered "Sold Items" page. */38export const SoldPagePayloadSchema = z.object({39 kind: z.literal('sold_page'),40 seed: SeedSchema,41 page: z.number().int(),42 pageUrl: z.string(),43 request: z.record(z.string(), z.unknown()),44 total: z.number().nullable(),45 lots: z.array(GoldinLotSchema),46 auctions: z.record(z.string(), z.object({ title: z.string().nullable().optional(), auction_type: z.string().nullable().optional(), end_timestamp: z.string().nullable().optional(), buyer_premium: z.number().nullable().optional() })).default({}),47});48export type SoldPagePayload = z.infer<typeof SoldPagePayloadSchema>;4950/** Raw payload for one rendered item page (lookup). Text is pre-flattened; no HTML stored. */51export const ItemPagePayloadSchema = z.object({52 kind: z.literal('item_page'),53 url: z.string(),54 title: z.string(),55 breadcrumb: z.array(z.string()).default([]),56 totalPriceText: z.string().nullable(),57 winningBidText: z.string().nullable(),58 endedText: z.string().nullable(),59 bidsText: z.string().nullable(),60 sold: z.boolean(),61 description: z.string().nullable(),62 images: z.array(z.string()).default([]),63 lotNumber: z.string().nullable(),64});65export type ItemPagePayload = z.infer<typeof ItemPagePayloadSchema>;6667const ConfigSchema = z.object({68 seeds: z.array(SeedSchema).default([]),69 pageSize: z.number().int().min(24).max(240).default(240),70 maxPagesPerSeed: z.number().int().min(1).default(40),71 renderingWaitMs: z.number().int().default(6000),72});7374function parseUtc(s: string | null | undefined): Date | null {75 if (!s) return null;76 const iso = /[zZ]$|[+-]\d{2}:?\d{2}$/.test(s) ? s : `${s}Z`;77 const d = new Date(iso);78 return Number.isNaN(d.getTime()) ? null : d;79}8081/**82 * Goldin titles end with " - <GRADER> <GRADE>" (e.g. "- PSA GEM MT 10"). Parse that suffix first so that83 * words like "Tag Team" inside the card name are not mistaken for TAG grading.84 */85export function parseGoldinGrade(title: string): ReturnType<typeof parseGradeFromTitle> {86 const parts = title.split(/\s+-\s+/);87 for (let i = parts.length - 1; i >= 1; i--) {88 const g = parseGradeFromTitle(parts[i]!);89 if (g.grader && g.grader !== 'raw' && g.grade) return g;90 }91 const g = parseGradeFromTitle(title);92 if (g.grader === 'tag' && !/\bTAG\s+\d/i.test(title)) return { grader: null, grade: null, qualifier: null };93 return g;94}9596export function soldPageUrl(seedPath: string, page: number, pageSize: number): string {97 return `${SITE}${seedPath}?page=${page}&number_of_lots=${pageSize}&show_only=Sold%20Items&sort=Most_Recent_Bids`;98}99100/** Pick the lots_v2 XHR that carries the seed filter (the page also fires an unfiltered call). */101export function extractLotsXhr(render: ScrapflyRenderResult): { request: Record<string, unknown>; total: number | null; lots: unknown[] } | null {102 const candidates = render.xhr.filter((x) => x.url.includes('/api/lots_v2') && x.responseBody);103 let best: { request: Record<string, unknown>; total: number | null; lots: unknown[] } | null = null;104 for (const c of candidates) {105 try {106 const req = (JSON.parse(c.requestBody ?? '{}') as { search?: Record<string, unknown> }).search ?? {};107 const body = JSON.parse(c.responseBody!) as { searchalgolia?: { lots?: unknown[]; total?: string | number } };108 const sa = body.searchalgolia;109 if (!sa?.lots) continue;110 const filtered = Boolean(req.sub_category || req.category || req.keyword);111 const total = sa.total === undefined ? null : Number(sa.total);112 const cand = { request: req, total: Number.isFinite(total as number) ? (total as number) : null, lots: sa.lots };113 if (filtered || !best) best = cand;114 } catch {115 /* skip malformed */116 }117 }118 return best;119}120121export function extractAuctionsXhr(render: ScrapflyRenderResult): SoldPagePayload['auctions'] {122 const out: SoldPagePayload['auctions'] = {};123 for (const x of render.xhr) {124 if (!x.url.endsWith('/api/auctions') || !x.responseBody || x.responseBody.length < 1000) continue;125 try {126 const body = JSON.parse(x.responseBody) as { auctions?: Array<Record<string, unknown>> };127 for (const a of body.auctions ?? []) {128 const id = a.auction_id as string | undefined;129 if (!id) continue;130 out[id] = { title: (a.title as string) ?? null, auction_type: (a.auction_type as string) ?? null, end_timestamp: (a.end_timestamp as string) ?? null, buyer_premium: typeof a.buyer_premium === 'number' ? a.buyer_premium : null };131 }132 } catch {133 /* ignore */134 }135 }136 return out;137}138139/** Flatten a rendered item page into a compact payload (no HTML persisted). */140export function parseItemPage(url: string, pageHtml: string): ItemPagePayload {141 const $ = H.load(pageHtml);142 $('script, style, noscript').remove();143 const text = $('body').text().replace(/\s+/g, ' ').trim();144 const title = $('h1').first().text().trim() || ($('title').text().split('|')[0] ?? '').trim();145 const winning = text.match(/\$([\d,]+(?:\.\d{2})?)\s*Winning Bid/i);146 // Live lots also embed a hidden "Lot Sold" template, so only the winning-bid figure proves a sale.147 const sold = winning !== null;148 // total price appears right before the winning bid on sold pages, or before "with Buyer’s Premium" on live ones149 let total: string | null = null;150 if (winning) {151 const before = text.slice(Math.max(0, winning.index! - 40), winning.index!);152 const m = before.match(/\$([\d,]+(?:\.\d{2})?)\s*$/);153 total = m ? `$${m[1]}` : null;154 } else {155 const m = text.match(/\$([\d,]+(?:\.\d{2})?)\s*with Buyer/i);156 total = m ? `$${m[1]}` : null;157 }158 const ended = text.match(/((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s*\d{1,2}\/\d{1,2}\/\d{2,4},\s*\d{1,2}:\d{2}\s*[AP]M\s*(?:P[SD]T|E[SD]T|C[SD]T|M[SD]T|UTC|GMT))/);159 const bids = text.match(/(?:^|\s|\|)(\d{1,4}) bids?\b/i);160 const lot = text.match(/Lot #(\d+)/);161 const descIdx = text.indexOf('Description');162 const description = descIdx >= 0 ? text.slice(descIdx + 'Description'.length, descIdx + 3000).split(/You are viewing a lot in|Want to Sell on Goldin/)[0]!.trim() : null;163 const crumbs: string[] = [];164 $('[data-testid^="lot-breadcrumb-"]').each((_, el) => {165 const t = $(el).text().trim();166 if (t) crumbs.push(t);167 });168 const images = new Set<string>();169 $('img[src*="cloudfront.net/public/Lots/"]').each((_, el) => {170 const src = $(el).attr('src');171 if (src) images.add(src);172 });173 return { kind: 'item_page', url, title, breadcrumb: crumbs, totalPriceText: total, winningBidText: winning ? `$${winning[1]}` : null, endedText: ended ? ended[1]! : null, bidsText: bids ? bids[1]! : null, sold, description, images: [...images].slice(0, 6), lotNumber: lot ? lot[1]! : null };174}175176/** Parse Goldin's display date "Sat, 11/17/12, 10:16 PM EDT" → UTC Date. */177export function parseGoldinDisplayDate(s: string | null | undefined): Date | null {178 if (!s) return null;179 const m = s.match(/(\d{1,2})\/(\d{1,2})\/(\d{2,4}),\s*(\d{1,2}):(\d{2})\s*([AP]M)\s*([A-Z]{2,4})/);180 if (!m) return null;181 const month = Number(m[1]);182 const day = Number(m[2]);183 let year = Number(m[3]);184 if (year < 100) year += 2000;185 let hour = Number(m[4]) % 12;186 if (m[6] === 'PM') hour += 12;187 const minute = Number(m[5]);188 const tzOffsets: Record<string, number> = { PDT: -7, PST: -8, EDT: -4, EST: -5, CDT: -5, CST: -6, MDT: -6, MST: -7, UTC: 0, GMT: 0 };189 const off = tzOffsets[m[7]!] ?? -7;190 return new Date(Date.UTC(year, month - 1, day, hour - off, minute));191}192193export default function createConnector(meta: ConnectorMeta) {194 return new GoldinConnector(meta);195}196197export class GoldinConnector extends BaseConnector {198 readonly version = '1.0.0';199 readonly parserVersion = '1.0.0';200 override readonly urlPatterns = [/^https?:\/\/(www\.)?goldin\.co\/item\/[^/?#]+/i];201 protected override minIntervalMs = 2000;202 private readonly config = ConfigSchema.parse(this.meta.config ?? {});203204 private apiKey(): string {205 const key = process.env.SCRAPFLY_API_KEY;206 if (!key) throw new Error('goldin connector requires SCRAPFLY_API_KEY');207 return key;208 }209210 private async render(ctx: CrawlContext, url: string): Promise<ScrapflyRenderResult> {211 await this.throttle();212 const stats = (ctx.engineStats.scrapfly ??= { attempts: 0, success: 0, credits: 0, ms: 0 });213 stats.attempts++;214 const res = await scrapflyRender(url, { apiKey: this.apiKey(), renderingWaitMs: this.config.renderingWaitMs, country: 'us' });215 stats.credits += res.cost;216 stats.ms += res.durationMs;217 if (res.ok) stats.success++;218 else ctx.anomaly('blocked_request', `${url}: ${res.error ?? `status ${res.status}`}`);219 return res;220 }221222 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {223 const seeds = ctx.options.categories?.length ? this.config.seeds.filter((s) => ctx.options.categories!.includes(s.categorySlug)) : this.config.seeds;224 const cursor = { ...(ctx.options.cursor ?? {}) } as Record<string, { lastEnd?: string }>;225 const pageSize = ctx.options.mode === 'probe' ? 24 : this.config.pageSize;226 let yielded = 0;227 for (const seed of seeds) {228 if (ctx.signal?.aborted) return;229 const lastEnd = ctx.options.mode === 'incremental' ? cursor[seed.path]?.lastEnd : undefined;230 let newestSeen: string | undefined;231 const maxPages = ctx.options.mode === 'probe' ? 1 : this.config.maxPagesPerSeed;232 for (let page = 1; page <= maxPages; page++) {233 const url = soldPageUrl(seed.path, page, pageSize);234 const render = await this.render(ctx, url);235 if (!render.ok) break;236 const xhr = extractLotsXhr(render);237 if (!xhr) {238 ctx.anomaly('empty_page', `${url}: no lots_v2 response captured`);239 break;240 }241 const lots = xhr.lots.map((l) => GoldinLotSchema.safeParse(l)).filter((r) => r.success).map((r) => r.data);242 if (lots.length !== xhr.lots.length) ctx.anomaly('parse_failure', `${xhr.lots.length - lots.length} lots failed schema on ${url}`);243 if (lots.length === 0) break;244 const auctions = extractAuctionsXhr(render);245 const usedAuctions: SoldPagePayload['auctions'] = {};246 for (const l of lots) if (l.auction_id && auctions[l.auction_id]) usedAuctions[l.auction_id] = auctions[l.auction_id]!;247 const payload: SoldPagePayload = { kind: 'sold_page', seed, page, pageUrl: url, request: xhr.request, total: xhr.total, lots, auctions: usedAuctions };248 const realEnds = lots.map((l) => parseUtc(l.end_timestamp)).filter((d): d is Date => d !== null && d.getTime() <= Date.now() + 86_400_000);249 const newest = realEnds.reduce<Date | null>((a, d) => (!a || d > a ? d : a), null);250 const oldest = realEnds.reduce<Date | null>((a, d) => (!a || d < a ? d : a), null);251 if (newest && (!newestSeen || newest.toISOString() > newestSeen)) newestSeen = newest.toISOString();252 yield { url, externalId: `${seed.path}#${page}`, kind: 'sale', engine: 'scrapfly', httpStatus: render.status, payload, fetchedAt: new Date() };253 yielded += lots.length;254 if (this.reached(ctx, yielded)) return;255 if (lastEnd && oldest && oldest.toISOString() <= lastEnd) break; // caught up with previous run256 if (xhr.total !== null && page * pageSize >= xhr.total) break;257 }258 if (newestSeen) {259 cursor[seed.path] = { lastEnd: newestSeen };260 await ctx.setCursor(cursor);261 }262 }263 }264265 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {266 const render = await this.render(ctx, url);267 if (!render.ok || !render.html) return [];268 const payload = parseItemPage(url, render.html);269 return [{ url, externalId: url.split('/item/')[1]?.split(/[?#]/)[0] ?? null, kind: 'sale', engine: 'scrapfly', httpStatus: render.status, payload, fetchedAt: new Date() }];270 }271272 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {273 const p = raw.payload as { kind?: string };274 if (p?.kind === 'item_page') return this.normalizeItem(raw, ItemPagePayloadSchema.parse(raw.payload));275 const page = SoldPagePayloadSchema.parse(raw.payload);276 const out: NormalizedRecord[] = [];277 const now = Date.now() + 86_400_000;278 for (const lot of page.lots) {279 if (lot.status && lot.status !== 'Completed_Sold') continue;280 const saleDate = parseUtc(lot.end_timestamp);281 if (!saleDate || saleDate.getTime() > now || lot.current_price === null || lot.current_price === undefined || lot.current_price <= 0) continue;282 const bp = lot.buyer_premium ?? page.auctions[lot.auction_id ?? '']?.buyer_premium ?? null;283 const price = bp !== null ? Math.round(lot.current_price * (1 + bp / 100) * 100) / 100 : lot.current_price;284 const grade = parseGoldinGrade(lot.title);285 const categorySlug = classifyGoldinTitle(lot.title, page.seed);286 const card = parseCardAttributes(lot.title, categorySlug);287 const auction = lot.auction_id ? page.auctions[lot.auction_id] : undefined;288 const auctionType = (lot.auction_type ?? auction?.auction_type ?? '').toLowerCase();289 const record = NormalizedSaleSchema.parse({290 kind: 'sale',291 connectorId: this.meta.id,292 sourceId: this.meta.sourceId,293 sourceUrl: `${SITE}/item/${lot.meta_slug}`,294 externalId: lot.lot_id,295 rawTitle: lot.title,296 description: null,297 imageUrls: lot.primary_image_name ? [`${IMAGE_CDN}/${lot.lot_id}/${lot.primary_image_name}@1x`] : [],298 attributes: {299 categorySlug,300 name: card.name ?? lot.title,301 brand: card.brand,302 set: card.set,303 number: card.number,304 year: extractYear(lot.title),305 variant: card.variant,306 identifiers: { goldin_lot_id: lot.lot_id, goldin_slug: lot.meta_slug },307 metadata: { auction_id: lot.auction_id ?? null, auction_title: auction?.title ?? null, auction_type: lot.auction_type ?? auction?.auction_type ?? null, hammer_price: lot.current_price, buyer_premium_pct: bp, lot_number: lot.lot_number ?? null, bids: lot.number_of_bids ?? null, goldin_seed: page.seed.path },308 },309 grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null },310 condition: {},311 observedAt: raw.fetchedAt,312 confidence: grade.grader ? 0.9 : 0.8,313 parserVersion: this.parserVersion,314 saleType: auctionType.includes('fixed') ? 'fixed_price' : 'auction',315 saleDate,316 price,317 currency: 'USD',318 buyerPremiumIncluded: bp !== null,319 quantity: 1,320 isBundle: isBundleTitle(lot.title),321 location: 'US',322 auctionHouse: 'Goldin',323 lotNumber: lot.lot_number !== null && lot.lot_number !== undefined ? String(lot.lot_number) : null,324 });325 out.push(record);326 }327 return out;328 }329330 private normalizeItem(raw: RawRecordLike, item: ItemPagePayload): NormalizedRecord[] {331 if (!item.sold) return [];332 const total = parsePrice(item.totalPriceText, 'USD');333 const hammer = parsePrice(item.winningBidText, 'USD');334 const saleDate = parseGoldinDisplayDate(item.endedText);335 const amount = total?.amount ?? hammer?.amount;336 if (!amount || !saleDate) return [];337 const grade = parseGoldinGrade(item.title);338 const cert = item.description?.match(/\b(?:PSA|BGS|CGC|SGC|Beckett)\b[^()]{0,40}\((\d{6,12})\)/i)?.[1] ?? item.description?.match(/\b(?:cert(?:ification)?(?:\s*(?:no|#|number))?)[:\s#]*(\d{6,12})/i)?.[1] ?? null;339 const seedGuess = { path: 'lookup', categorySlug: item.breadcrumb.map((b) => b.toLowerCase()).includes('pokemon') ? 'pokemon' : 'sports_memorabilia' };340 const categorySlug = classifyGoldinTitle(item.title, seedGuess, item.breadcrumb);341 const card = parseCardAttributes(item.title, categorySlug);342 return [343 NormalizedSaleSchema.parse({344 kind: 'sale',345 connectorId: this.meta.id,346 sourceId: this.meta.sourceId,347 sourceUrl: item.url,348 externalId: raw.externalId ?? item.url,349 rawTitle: item.title,350 description: item.description,351 imageUrls: item.images,352 attributes: { categorySlug, name: card.name ?? item.title, brand: card.brand, set: card.set, number: card.number, year: extractYear(item.title), variant: card.variant, identifiers: { goldin_slug: raw.externalId ?? '' }, metadata: { hammer_price: hammer?.amount ?? null, breadcrumb: item.breadcrumb, lot_number: item.lotNumber } },353 grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: cert },354 condition: {},355 observedAt: raw.fetchedAt,356 confidence: 0.85,357 parserVersion: this.parserVersion,358 saleType: 'auction',359 saleDate,360 price: amount,361 currency: 'USD',362 buyerPremiumIncluded: total ? true : null,363 quantity: 1,364 isBundle: isBundleTitle(item.title),365 location: 'US',366 auctionHouse: 'Goldin',367 lotNumber: item.lotNumber,368 }),369 ];370 }371}372