TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { CurrencyCode, NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { makeSale } from '../../firecrawl/_carlib/index.js';5import { isNumisBundle, numisAttributes, numisCategory, parseCoinGrade, type NumisSlug } from '../../firecrawl/_g5-numismatics-lib/index.js';67/**8 * Stanley Gibbons Baldwin's (London) — stamps, coins, medals. The Next.js site exposes its page data as9 * JSON (/_next/data/<buildId>/…): the past-auctions list and, per auction, every lot with estimate,10 * hammer price, buyer's premium amount and status. Hammer and premium are therefore explicit.11 */1213const SITE = 'https://sgbaldwins.com';14const PARSER_VERSION = '1.0.0';15const CHUNK = 200;16const DEPARTMENTS: Record<string, NumisSlug> = { Stamps: 'stamps', Coins: 'coins', 'Medals & Militaria': 'medals' };1718export const AuctionSchema = z.object({19 uuid: z.string(),20 saleNumber: z.string().nullable(),21 title: z.string(),22 department: z.string().nullable(),23 slug: z.string(),24 amsDate: z.string().nullable(),25 dateTimeText: z.string().nullable(),26 status: z.string().nullable(),27 currencySymbol: z.string().nullable(),28 premiums: z.array(z.object({ percent: z.number(), amount_over: z.number().nullable().optional() })).default([]),29});30export const LotSchema = z.object({31 uuid: z.string(),32 lotNo: z.string(),33 title: z.string(),34 country: z.string().nullable(),35 low: z.string().nullable(),36 high: z.string().nullable(),37 startPrice: z.string().nullable(),38 hammerPrice: z.string().nullable(),39 premium: z.string().nullable(),40 status: z.string().nullable(),41 sessionDate: z.string().nullable(),42 categoryName: z.string().nullable(),43 images: z.array(z.string()).default([]),44});45export const PayloadSchema = z.object({ kind: z.literal('auction_lots'), auction: AuctionSchema, url: z.string(), chunk: z.number().int(), totalLots: z.number().int(), lots: z.array(LotSchema) });46export type Payload = z.infer<typeof PayloadSchema>;4748export interface ListEntry {49 uuid: string;50 saleNumber: string | null;51 title: string;52 department: string | null;53 slug: string;54 amsDate: string | null;55}5657export function buildIdFromHtml(htmlText: string): string | null {58 const nd = H.nextData(htmlText) as { buildId?: string } | null;59 return nd?.buildId ?? htmlText.match(/"buildId":"([^"]+)"/)?.[1] ?? null;60}6162/** past-auctions.json → auctions (newest first as served). */63export function parsePastAuctions(json: unknown): ListEntry[] {64 const list = (json as { pageProps?: { auctionsListData?: Array<Record<string, unknown>> } })?.pageProps?.auctionsListData ?? [];65 const out: ListEntry[] = [];66 for (const a of list) {67 const uuid = typeof a.auctionId === 'string' ? a.auctionId : null;68 const slug = typeof a.arrowLink === 'string' ? a.arrowLink : null;69 if (!uuid || !slug || typeof a.title !== 'string') continue;70 out.push({ uuid, saleNumber: typeof a.saleNumber === 'string' ? a.saleNumber : null, title: a.title, department: typeof a.department === 'string' ? a.department : null, slug, amsDate: typeof a.amsDate === 'string' ? a.amsDate : null });71 }72 return out;73}7475/** auction page JSON → auction header + lots. */76export function parseAuctionJson(json: unknown, entry: ListEntry): { auction: z.infer<typeof AuctionSchema>; lots: z.infer<typeof LotSchema>[] } | null {77 const pp = (json as { pageProps?: Record<string, unknown> })?.pageProps;78 if (!pp || !Array.isArray(pp.lots)) return null;79 const ams = (pp.amsData ?? {}) as { status?: string; premiums?: Array<{ percent: number; amount_over?: number | null }> };80 const page = (pp.auctionPageData ?? {}) as { attributes?: { dateTime?: string } };81 const cats = new Map<string, string>();82 for (const c of (pp.categories as Array<{ uuid?: string; name?: string }> | undefined) ?? []) if (c.uuid && c.name) cats.set(c.uuid, c.name);83 const lots: z.infer<typeof LotSchema>[] = [];84 let symbol: string | null = null;85 for (const raw of pp.lots as Array<Record<string, unknown>>) {86 const title = (raw.title as { en?: string } | undefined)?.en?.trim();87 const uuid = typeof raw.uuid === 'string' ? raw.uuid : null;88 if (!uuid || !title) continue;89 const fin = (raw.financeItem ?? {}) as { hammer_price?: string | null; premium?: string | null };90 symbol ??= (raw.auction as { currency?: { symbol?: string } } | undefined)?.currency?.symbol ?? null;91 const images = ((raw.images as Array<{ data?: { original?: { url?: string }; sm?: { url?: string } } }> | undefined) ?? []).map((i) => i.data?.original?.url ?? i.data?.sm?.url).filter((u): u is string => Boolean(u)).slice(0, 3);92 lots.push({93 uuid,94 lotNo: String(raw.lot_no ?? ''),95 title,96 country: (typeof raw.country === 'string' ? raw.country : null) ?? (raw.dynamic_fields as { en?: { country?: string } } | undefined)?.en?.country ?? null,97 low: str(raw.low),98 high: str(raw.high),99 startPrice: str(raw.start_price),100 hammerPrice: str(fin.hammer_price ?? raw.hammer_price),101 premium: str(fin.premium),102 status: typeof raw.status === 'string' ? raw.status : null,103 sessionDate: typeof raw.session_date === 'string' ? raw.session_date : null,104 categoryName: typeof raw.category_uuid === 'string' ? (cats.get(raw.category_uuid) ?? null) : null,105 images,106 });107 }108 return {109 auction: { uuid: entry.uuid, saleNumber: entry.saleNumber, title: entry.title, department: entry.department, slug: entry.slug, amsDate: entry.amsDate, dateTimeText: page.attributes?.dateTime ?? null, status: ams.status ?? null, currencySymbol: symbol, premiums: ams.premiums ?? [] },110 lots,111 };112}113114function str(v: unknown): string | null {115 if (v === null || v === undefined || v === '') return null;116 return typeof v === 'number' ? String(v) : typeof v === 'string' ? v : null;117}118119export function currencyFromSymbol(sym: string | null): CurrencyCode {120 if (!sym) return 'GBP';121 if (sym.includes('$')) return 'USD';122 if (sym.includes('€')) return 'EUR';123 return 'GBP';124}125126interface Cursor {127 doneAuctions?: string[];128 done?: boolean;129 updatedAt?: string;130}131132export class SgBaldwinsConnector extends BaseConnector {133 readonly version = '1.0.0';134 readonly parserVersion = PARSER_VERSION;135 protected override minIntervalMs = 1500;136 override readonly urlPatterns = [/sgbaldwins\.com\/auctions\//i];137138 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {139 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 3);140 const departments = (this.meta.config.departments as string[] | undefined) ?? Object.keys(DEPARTMENTS);141 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };142 const done = new Set(cursor.doneAuctions ?? []);143 await this.throttle();144 const home = await ctx.fetch(`${SITE}/`, { engines: ['api'], responseType: 'text', minQuality: 0, force: true });145 const buildId = home.success && home.html ? buildIdFromHtml(home.html) : null;146 if (!buildId) {147 ctx.anomaly('selector_missing', `home: __NEXT_DATA__ buildId not found (${home.error ?? home.httpStatus})`);148 return;149 }150 await this.throttle();151 const listUrl = `${SITE}/_next/data/${buildId}/auctions/past-auctions.json`;152 const list = await ctx.fetch(listUrl, { engines: ['api'], minQuality: 0 });153 const auctions = list.success && list.json ? parsePastAuctions(list.json) : [];154 if (!auctions.length) {155 ctx.anomaly(list.success ? 'schema_drift' : 'page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus ?? 'no auctionsListData'}`);156 return;157 }158 const wanted = auctions.filter((a) => a.department && departments.includes(a.department));159 let processed = 0;160 let yielded = 0;161 for (const entry of wanted) {162 if (ctx.signal?.aborted || processed >= auctionsPerRun || done.has(entry.uuid) || this.reached(ctx, yielded)) {163 if (done.has(entry.uuid)) continue;164 break;165 }166 const url = `${SITE}/_next/data/${buildId}${entry.slug}.json`;167 await this.throttle();168 const res = await ctx.fetch(url, { engines: ['api'], expect: ['title', 'price'], parse: (r) => {169 const p = parseAuctionJson(r.json, entry);170 const sold = p?.lots.find((l) => l.hammerPrice);171 return p ? { title: p.lots[0]?.title ?? entry.title, price: sold?.hammerPrice ?? null } : null;172 } });173 processed++;174 if (!res.success || !res.json) {175 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);176 continue;177 }178 const parsed = parseAuctionJson(res.json, entry);179 if (!parsed) {180 ctx.anomaly('schema_drift', `${url}: pageProps.lots missing`);181 continue;182 }183 if (parsed.auction.status && parsed.auction.status !== 'completed') continue; // still running184 for (let i = 0; i < parsed.lots.length; i += CHUNK) {185 const lots = parsed.lots.slice(i, i + CHUNK);186 if (!lots.some((l) => l.hammerPrice)) continue;187 yielded++;188 const payload: Payload = { kind: 'auction_lots', auction: parsed.auction, url: `${SITE}${entry.slug}`, chunk: i / CHUNK, totalLots: parsed.lots.length, lots };189 yield { url: `${SITE}${entry.slug}`, externalId: `auction:${entry.uuid}:chunk:${i / CHUNK}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };190 }191 done.add(entry.uuid);192 if (ctx.options.mode === 'backfill') await ctx.progress({ page: wanted.indexOf(entry) + 1, totalPages: wanted.length, itemsProcessed: yielded, reachedDate: entry.amsDate ? new Date(entry.amsDate) : null });193 await ctx.setCursor({ doneAuctions: [...done].slice(-400), updatedAt: new Date().toISOString() });194 }195 if (ctx.options.mode === 'backfill' && wanted.every((a) => done.has(a.uuid))) await ctx.setCursor({ doneAuctions: [...done].slice(-400), done: true, updatedAt: new Date().toISOString() });196 }197198 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {199 const p = PayloadSchema.parse(raw.payload);200 const currency = currencyFromSymbol(p.auction.currencySymbol);201 const fallback: NumisSlug = DEPARTMENTS[p.auction.department ?? ''] ?? 'coins';202 const out: NormalizedSale[] = [];203 for (const lot of p.lots) {204 const hammer = Number(lot.hammerPrice);205 if (!lot.hammerPrice || !Number.isFinite(hammer) || hammer <= 0 || lot.status === 'unsold') continue;206 const dateSrc = lot.sessionDate ?? p.auction.amsDate;207 if (!dateSrc) continue;208 const d = new Date(dateSrc);209 if (Number.isNaN(d.getTime())) continue;210 const saleDate = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));211 const hint = `${p.auction.department ?? ''} ${lot.categoryName ?? ''}`;212 const categorySlug = fallback === 'stamps' ? 'stamps' : numisCategory(lot.title, hint, fallback);213 const g = parseCoinGrade(lot.title);214 const premium = lot.premium ? Number(lot.premium) : null;215 const attributes = numisAttributes({216 categorySlug,217 title: lot.title,218 section: lot.categoryName ?? p.auction.title,219 country: null,220 identifiers: { sgbaldwins_lot: lot.uuid, ...(lot.title.match(/\bSG\s?(\d+[a-z]?)\b/) && categorySlug === 'stamps' ? { sg_number: lot.title.match(/\bSG\s?(\d+[a-z]?)\b/)![1]! } : {}) },221 metadata: {222 sale_number: p.auction.saleNumber,223 auction_title: p.auction.title,224 department: p.auction.department,225 category: lot.categoryName,226 country_label: lot.country,227 estimate_low: lot.low ? Number(lot.low) : null,228 estimate_high: lot.high ? Number(lot.high) : null,229 start_price: lot.startPrice ? Number(lot.startPrice) : null,230 hammer_price: hammer,231 buyer_premium_amount: premium,232 buyer_premium_percent: p.auction.premiums[0]?.percent ?? null,233 total_with_premium: premium !== null ? hammer + premium : null,234 },235 });236 if (!attributes.country && lot.country) attributes.country = countryLabel(lot.country);237 const sale = makeSale({238 meta: this.meta,239 sourceUrl: `${SITE}${p.auction.slug}`,240 externalId: lot.uuid,241 rawTitle: lot.title,242 attributes,243 price: hammer,244 currency,245 saleDate,246 buyerPremiumIncluded: false,247 auctionHouse: "Stanley Gibbons Baldwin's",248 lotNumber: lot.lotNo || null,249 imageUrls: lot.images,250 observedAt: raw.fetchedAt,251 parserVersion: PARSER_VERSION,252 confidence: 0.9,253 isBundle: isNumisBundle(lot.title),254 conditionRaw: g.conditionRaw,255 location: 'GB',256 });257 sale.grade = { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: g.certificationNumber };258 out.push(sale);259 }260 return out;261 }262}263264const COUNTRY_LABELS: Record<string, string> = { 'Great Britain': 'GB', 'United Kingdom': 'GB', England: 'GB', Scotland: 'GB', Ireland: 'IE', Australia: 'AU', Canada: 'CA', 'United States': 'US', USA: 'US', India: 'IN', France: 'FR', Germany: 'DE', Italy: 'IT', Spain: 'ES', 'South Africa': 'ZA', 'New Zealand': 'NZ', 'Hong Kong': 'HK', China: 'CN', Japan: 'JP', Switzerland: 'CH', Netherlands: 'NL', Belgium: 'BE', Austria: 'AT', Russia: 'RU', Malta: 'MT', Cyprus: 'CY', Gibraltar: 'GI', Ceylon: 'LK', 'Sri Lanka': 'LK', Jamaica: 'JM', Bermuda: 'BM' };265function countryLabel(label: string): string | null {266 return COUNTRY_LABELS[label.trim()] ?? null;267}268269export default (meta: ConnectorMeta) => new SgBaldwinsConnector(meta);270