TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { makeSale } from '../../firecrawl/_carlib/index.js';5import { isNumisBundle, numisAttributes, numisCategory } from '../../firecrawl/_g5-numismatics-lib/index.js';67/**8 * David Lawrence Rare Coins — sold auction lots from the public Collectibles Showcase JSON API that the9 * davidlawrence.com SPA itself calls (no key: the tenant id is resolved from /consumer/client?url=…).10 * Every lot carries grading service, grade, certification number, PCGS number and sale price in cents.11 * DLRC states "DLRC has no buyer's fee" → the sale price is the full amount paid.12 */1314const API = 'https://api.collectiblesshowcase.com';15const SITE = 'https://www.davidlawrence.com';16const PARSER_VERSION = '1.0.0';17const PAGE_SIZE = 100;1819export const LotSchema = z.object({20 id: z.number(),21 auctionId: z.number().nullable(),22 lotId: z.number().nullable(),23 lotNumber: z.number().nullable(),24 totalBids: z.number().nullable().optional(),25 lotCurrentBid: z.number().nullable().optional(),26 lotClosesAt: z.string().nullable(),27 soldInAuction: z.boolean().nullable().optional(),28 itemType: z.string().nullable().optional(),29 name: z.string(),30 shortDescription: z.string().nullable().optional(),31 description: z.string().nullable().optional(),32 seriesName: z.string().nullable().optional(),33 groupName: z.string().nullable().optional(),34 categoryName: z.string().nullable().optional(),35 certificationNumber: z.string().nullable().optional(),36 gradingService: z.string().nullable().optional(),37 grade: z.number().nullable().optional(),38 fullGrade: z.string().nullable().optional(),39 isPlus: z.boolean().nullable().optional(),40 isCac: z.boolean().nullable().optional(),41 isEPQ: z.boolean().nullable().optional(),42 price: z.number().nullable().optional(),43 salePrice: z.number().nullable().optional(),44 status: z.string().nullable().optional(),45 catalogEntry: z46 .object({47 id: z.number().nullable().optional(),48 pcgsNumber: z.number().nullable().optional(),49 title: z.string().nullable().optional(),50 denomination: z.string().nullable().optional(),51 coinDate: z.number().nullable().optional(),52 mintMark: z.string().nullable().optional(),53 designation: z.string().nullable().optional(),54 majorVariety: z.string().nullable().optional(),55 dieVariety: z.string().nullable().optional(),56 strikeType: z.string().nullable().optional(),57 mintage: z.number().nullable().optional(),58 numistaTypeId: z.union([z.number(), z.string()]).nullable().optional(),59 kmNumber: z.string().nullable().optional(),60 categoryName: z.string().nullable().optional(),61 })62 .nullable()63 .optional(),64 images: z.array(z.object({ url: z.string().nullable().optional(), thumbnailUrl: z.string().nullable().optional() })).default([]),65});66export type Lot = z.infer<typeof LotSchema>;67export const PayloadSchema = z.object({68 kind: z.literal('past_auction_page'),69 url: z.string(),70 auctionId: z.number().nullable(),71 auctionTitle: z.string().nullable(),72 pageNumber: z.number().int(),73 totalPages: z.number().int().nullable(),74 totalItems: z.number().int().nullable(),75 lots: z.array(LotSchema),76});77export type Payload = z.infer<typeof PayloadSchema>;7879const ApiPage = z.object({ pagination: z.object({ totalItemsCount: z.number(), pageSize: z.number(), pageNumber: z.number(), totalPages: z.number() }).nullable(), payload: z.array(z.unknown()) });80const PastAuctions = z.object({ payload: z.array(z.object({ id: z.number(), title: z.string(), estimatedClosesAt: z.string().nullable(), totalLotCount: z.number().nullable() })) });8182/** "1793 1/2C NGC/CAC Poor 01" + structured fields → grader slug, grade token. */83export function dlrcGrade(lot: Pick<Lot, 'gradingService' | 'fullGrade' | 'grade' | 'isPlus' | 'isCac' | 'certificationNumber' | 'name'>): { grader: string | null; grade: string | null; qualifier: string | null; certificationNumber: string | null } {84 const svc = lot.gradingService?.toUpperCase() ?? null;85 const grader = svc && /^(PCGS|NGC|ANACS|ICCS|PMG|ICG|CAC)$/.test(svc) ? svc.toLowerCase() : svc ? svc.toLowerCase() : null;86 let grade = lot.fullGrade ?? (lot.grade ? String(lot.grade) : null);87 if (grade && lot.isPlus && !grade.includes('+')) grade = grade.replace(/^([A-Z]+\d{1,2})/, '$1+');88 const details = /details/i.test(lot.name) ? lot.name.match(/\b(\w+ Details(?: \([^)]*\))?)/i)?.[1] ?? 'Details' : null;89 const qualifier = [lot.isCac ? 'CAC' : null, details].filter(Boolean).join(' · ') || null;90 return { grader, grade, qualifier, certificationNumber: lot.certificationNumber ?? null };91}9293interface Cursor {94 doneAuctions?: number[];95 inProgress?: { auctionId: number | null; title: string | null; pageNumber: number; totalPages: number | null } | null;96 /** backfill over the unfiltered archive (includes pre-2023 legacy lots without an auction id) */97 archivePage?: number;98 done?: boolean;99 updatedAt?: string;100}101102export class DlrcConnector extends BaseConnector {103 readonly version = '1.0.0';104 readonly parserVersion = PARSER_VERSION;105 protected override minIntervalMs = 1500;106 override readonly urlPatterns = [/davidlawrence\.com\//i];107 private clientId: string | null = null;108109 private async headers(ctx: CrawlContext): Promise<Record<string, string> | null> {110 if (this.clientId) return { 'x-client-id': this.clientId };111 const configured = this.meta.config.clientId;112 if (configured) this.clientId = String(configured);113 else {114 await this.throttle();115 const res = await ctx.fetch(`${API}/consumer/client?url=${encodeURIComponent(SITE)}`, { engines: ['api'], minQuality: 0, force: true });116 const id = (res.json as { payload?: { id?: number } } | null)?.payload?.id;117 if (!res.success || !id) {118 ctx.anomaly('page_fetch_failed', `client id lookup: ${res.error ?? res.httpStatus}`);119 return null;120 }121 this.clientId = String(id);122 }123 return { 'x-client-id': this.clientId };124 }125126 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {127 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);128 const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 40);129 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };130 const headers = await this.headers(ctx);131 if (!headers) return;132 let pages = 0;133 let yielded = 0;134135 const fetchLots = async (params: Record<string, string | number>) => {136 const q = new URLSearchParams(Object.entries(params).map(([k, v]) => [k, String(v)] as [string, string]));137 const url = `${API}/consumer/inventory/past/auction?${q}`;138 await this.throttle();139 const res = await ctx.fetch(url, { engines: ['api'], headers, expect: ['title', 'price', 'identifiers'], parse: (r) => {140 const first = (r.json as { payload?: Array<Record<string, unknown>> } | null)?.payload?.[0];141 return first ? { title: first.name, price: first.salePrice, identifiers: first.certificationNumber ? { cert: first.certificationNumber } : null } : null;142 } });143 pages++;144 if (!res.success || !res.json) {145 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);146 return null;147 }148 const parsed = ApiPage.safeParse(res.json);149 if (!parsed.success) {150 ctx.anomaly('schema_drift', `${url}: ${parsed.error.issues[0]?.message}`);151 return null;152 }153 return { url, res, page: parsed.data };154 };155156 if (ctx.options.mode === 'backfill') {157 // Whole archive, catalogue order (includes legacy 2016+ sales without an auction id).158 let pageNumber = Math.max(1, cursor.archivePage ?? 1);159 let totalPages: number | null = null;160 while (pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, yielded)) {161 const got = await fetchLots({ pageNumber, pageSize: PAGE_SIZE });162 if (!got) break;163 totalPages = got.page.pagination?.totalPages ?? totalPages;164 const lots = got.page.payload;165 if (lots.length) {166 yielded++;167 const payload: Payload = { kind: 'past_auction_page', url: got.url, auctionId: null, auctionTitle: null, pageNumber, totalPages, totalItems: got.page.pagination?.totalItemsCount ?? null, lots: lots.map((l) => LotSchema.parse(l)) };168 yield { url: got.url, externalId: `archive:p${pageNumber}`, kind: 'sale', engine: got.res.engine, httpStatus: got.res.httpStatus, payload, fetchedAt: got.res.fetchedAt };169 }170 const finished = !lots.length || (totalPages !== null && pageNumber >= totalPages);171 pageNumber++;172 await ctx.progress({ page: pageNumber - 1, totalPages, itemsProcessed: yielded });173 await ctx.setCursor({ ...cursor, archivePage: pageNumber, done: finished, updatedAt: new Date().toISOString() });174 if (finished) return;175 }176 return;177 }178179 // Incremental: newest closed auctions not yet done, lots filtered per auction.180 await this.throttle();181 const list = await ctx.fetch(`${API}/consumer/auction/past`, { engines: ['api'], headers, minQuality: 0 });182 pages++;183 const parsedList = list.success ? PastAuctions.safeParse(list.json) : null;184 if (!parsedList?.success) {185 ctx.anomaly('page_fetch_failed', `/consumer/auction/past: ${list.error ?? list.httpStatus}`);186 return;187 }188 const done = new Set(cursor.doneAuctions ?? []);189 const auctions = [...parsedList.data.payload].sort((a, b) => (b.estimatedClosesAt ?? '').localeCompare(a.estimatedClosesAt ?? ''));190 const queue = [...(cursor.inProgress ? [cursor.inProgress] : []), ...auctions.filter((a) => !done.has(a.id) && a.id !== cursor.inProgress?.auctionId).map((a) => ({ auctionId: a.id as number | null, title: a.title as string | null, pageNumber: 1, totalPages: null as number | null }))];191 let finished = 0;192 for (const state of queue) {193 if (ctx.signal?.aborted || finished >= auctionsPerRun || pages >= pagesPerRun || this.reached(ctx, yielded)) break;194 let complete = false;195 while (pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, yielded)) {196 const got = await fetchLots({ auctions: state.auctionId ?? '', pageNumber: state.pageNumber, pageSize: PAGE_SIZE });197 if (!got) break;198 state.totalPages = got.page.pagination?.totalPages ?? state.totalPages;199 const lots = got.page.payload;200 if (lots.length) {201 yielded++;202 const payload: Payload = { kind: 'past_auction_page', url: got.url, auctionId: state.auctionId, auctionTitle: state.title, pageNumber: state.pageNumber, totalPages: state.totalPages, totalItems: got.page.pagination?.totalItemsCount ?? null, lots: lots.map((l) => LotSchema.parse(l)) };203 yield { url: got.url, externalId: `auction:${state.auctionId}:p${state.pageNumber}`, kind: 'sale', engine: got.res.engine, httpStatus: got.res.httpStatus, payload, fetchedAt: got.res.fetchedAt };204 }205 if (!lots.length || (state.totalPages !== null && state.pageNumber >= state.totalPages)) {206 complete = true;207 break;208 }209 state.pageNumber++;210 cursor.inProgress = state;211 await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-400), updatedAt: new Date().toISOString() });212 }213 if (complete) {214 if (state.auctionId !== null) done.add(state.auctionId);215 finished++;216 cursor.inProgress = null;217 } else cursor.inProgress = state;218 await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-400), updatedAt: new Date().toISOString() });219 }220 }221222 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {223 const p = PayloadSchema.parse(raw.payload);224 const out: NormalizedSale[] = [];225 for (const lot of p.lots) {226 const cents = lot.salePrice ?? (lot.soldInAuction ? lot.lotCurrentBid ?? null : null);227 if (!cents || cents <= 0 || (lot.status && lot.status !== 'SOLD' && !lot.soldInAuction)) continue;228 const closes = lot.lotClosesAt ? new Date(lot.lotClosesAt) : null;229 if (!closes || Number.isNaN(closes.getTime())) continue;230 const saleDate = new Date(Date.UTC(closes.getUTCFullYear(), closes.getUTCMonth(), closes.getUTCDate()));231 const ce = lot.catalogEntry ?? null;232 const categorySlug = lot.itemType === 'CURRENCY' || /currency|banknote|paper money/i.test(`${lot.categoryName ?? ''} ${lot.groupName ?? ''}`) ? 'banknotes' : numisCategory(lot.name, lot.categoryName ?? lot.groupName, 'coins');233 const g = dlrcGrade(lot);234 const identifiers: Record<string, string> = { dlrc_inventory_id: String(lot.id) };235 if (ce?.pcgsNumber) identifiers.pcgs_number = String(ce.pcgsNumber);236 if (g.certificationNumber && g.grader) identifiers[`${g.grader}_cert`] = g.certificationNumber;237 if (ce?.numistaTypeId) identifiers.numista_id = String(ce.numistaTypeId);238 const attributes = numisAttributes({239 categorySlug,240 title: lot.name,241 section: lot.seriesName ?? lot.groupName ?? null,242 series: lot.seriesName ?? null,243 country: lot.itemType === 'US_COIN' || lot.categoryName === 'U.S. Coins' ? 'US' : undefined,244 identifiers,245 metadata: {246 auction_id: lot.auctionId,247 auction_title: p.auctionTitle,248 lot_id: lot.lotId,249 item_type: lot.itemType,250 group: lot.groupName,251 category: lot.categoryName,252 catalog_title: ce?.title ?? null,253 designation: ce?.designation ?? null,254 strike_type: ce?.strikeType ?? null,255 major_variety: ce?.majorVariety || null,256 die_variety: ce?.dieVariety || null,257 mintage: ce?.mintage ?? null,258 total_bids: lot.totalBids ?? null,259 is_cac: lot.isCac ?? null,260 buyer_premium: "none — 'DLRC has no buyer's fee' (site notice); sale price is the total paid",261 },262 });263 if (ce?.coinDate && !attributes.year) attributes.year = ce.coinDate;264 if (ce?.mintMark && !attributes.variant) attributes.variant = `${ce.mintMark} mint`;265 if (ce?.mintMark) attributes.metadata.mint_mark = ce.mintMark;266 if (ce?.denomination) attributes.metadata.denomination = ce.denomination;267 const sale = makeSale({268 meta: this.meta,269 sourceUrl: `${SITE}/inventory/${lot.id}`,270 externalId: String(lot.id),271 rawTitle: lot.name,272 description: [lot.shortDescription, lot.description].filter(Boolean).join(' — ') || null,273 attributes,274 price: cents / 100,275 currency: 'USD',276 saleDate,277 buyerPremiumIncluded: true,278 auctionHouse: 'David Lawrence Rare Coins',279 lotNumber: lot.lotNumber !== null ? String(lot.lotNumber) : null,280 imageUrls: lot.images.map((i) => i.url ?? i.thumbnailUrl).filter((u): u is string => Boolean(u)).slice(0, 4),281 observedAt: raw.fetchedAt,282 parserVersion: PARSER_VERSION,283 confidence: g.certificationNumber ? 0.95 : 0.85,284 isBundle: isNumisBundle(lot.name),285 location: 'US',286 });287 sale.grade = g;288 out.push(sale);289 }290 return out;291 }292}293294export default (meta: ConnectorMeta) => new DlrcConnector(meta);295